{ "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "legalcode_url": "https://creativecommons.org/licenses/by-sa/4.0/legalcode.en", "adapter_license": "CC-BY-SA-4.0", "adapter": "whatmodel.sh", "creators": [ "Naman Jain", "King Han", "Alex Gu", "Wen-Ding Li", "Fanjia Yan", "Tianjun Zhang", "Sida Wang", "Armando Solar-Lezama", "Koushik Sen", "Ion Stoica" ], "title": "LiveCodeBench official website results — selected whatmodel.sh adaptation", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "source_releases": [ "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json" ], "changes": [ "Selected exact catalogue model/config matches and one explicit official slider window per model.", "Reproduced upstream mean and one-decimal rounding, then converted percentages to fractions.", "Mapped identifiers and added settings limitations, source pins and per-problem provenance." ], "sharealike": "This separate adapted numerical tranche, including any adapter copyright/database rights in it, is offered under CC BY-SA 4.0. Do not remove or supersede that licence with a blanket permissive licence.", "publication": "Publish lcb-sharealike.json, its provenance, NOTICE and full licence alongside the API; preserve row licensing/source_id/evidence linkage in displayed and API-derived representations. No additional downstream restrictions or DRM.", "database_scope": "Keep this separately attributable adapted database available. CC BY-SA 4.0 section 4 can also apply to database adaptations; physical separation alone does not waive those obligations." }, "sources": [ { "id": "livecodebench-website-20241226", "name": "LiveCodeBench official website results (2024-12-26)", "kind": "dataset", "match": [ "github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json" ], "exact": true, "license": "CC-BY-SA-4.0", "license_url": "/licenses/lcb-sharealike-NOTICE.txt", "terms_url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/README.md", "verdict": "allowed", "checked": "2026-09-25", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "quote": "This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License.", "adapter_license": "CC-BY-SA-4.0", "adapter_license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json", "provenance_file": "/data/lcb-sharealike-provenance.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "license_scope": "Numerical website release files in this same repository; not inferred from the MIT harness.", "publication_requirement": "Retain this tranche, attribution, changes, licence and provenance when displaying or serving its rows.", "icon": "huggingface.co" }, { "id": "livecodebench-website-20250801", "name": "LiveCodeBench official website results (2025-08-01)", "kind": "dataset", "match": [ "github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json" ], "exact": true, "license": "CC-BY-SA-4.0", "license_url": "/licenses/lcb-sharealike-NOTICE.txt", "terms_url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/README.md", "verdict": "allowed", "checked": "2026-09-25", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "quote": "This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License.", "adapter_license": "CC-BY-SA-4.0", "adapter_license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json", "provenance_file": "/data/lcb-sharealike-provenance.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "license_scope": "Numerical website release files in this same repository; not inferred from the MIT harness.", "publication_requirement": "Retain this tranche, attribution, changes, licence and provenance when displaying or serving its rows.", "icon": "huggingface.co" } ], "benchmarks": [ { "id": "livecodebench-official-20240101-20241101", "name": "LiveCodeBench official (Jan 1–Nov 1, 2024)", "kind": "acc", "category": "llm", "url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "family": "livecodebench-codegeneration", "description": "Official website code-generation pass@1. Inclusive UTC filter 2024-01-01 00:00:00 through 2024-11-01 00:00:00; 387 identical problem IDs per selected model. Mean then one-decimal percentage rounding as upstream. No difficulty components or repaired/composite scores.", "window_start": "2024-01-01T00:00:00Z", "window_end": "2024-11-01T00:00:00Z", "window_end_inclusive": true, "problem_count": 387, "release_commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70" }, { "id": "livecodebench-official-20240801-20250501", "name": "LiveCodeBench official (Aug 1, 2024–May 1, 2025)", "kind": "acc", "category": "llm", "url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "family": "livecodebench-codegeneration", "description": "Official website code-generation pass@1. Inclusive UTC filter 2024-08-01 00:00:00 through 2025-05-01 00:00:00; 454 identical problem IDs per selected model. Mean then one-decimal percentage rounding as upstream. No difficulty components or repaired/composite scores.", "window_start": "2024-08-01T00:00:00Z", "window_end": "2025-05-01T00:00:00Z", "window_end_inclusive": true, "problem_count": 454, "release_commit": "f186ab5041d6b768733b8ae8145151c867942134" }, { "id": "livecodebench-official-20231101-20250201", "name": "LiveCodeBench official release_v5 (Nov 1, 2023–Feb 1, 2025)", "kind": "acc", "category": "llm", "url": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "family": "livecodebench-codegeneration", "description": "Official website code-generation pass@1 from the release_v5 data file (build/v5.json). Inclusive UTC filter 2023-11-01 00:00:00 through 2025-02-01 00:00:00 (the v5 page's default slider bounds); 629 identical problem IDs per selected model. Mean then one-decimal percentage rounding as upstream. No difficulty components or repaired/composite scores. Only models with no other LiveCodeBench window are recorded here.", "window_start": "2023-11-01T00:00:00Z", "window_end": "2025-02-01T00:00:00Z", "window_end_inclusive": true, "problem_count": 629, "release_commit": "f186ab5041d6b768733b8ae8145151c867942134" } ], "results": [ { "model": "deepseek-v3", "bench": "livecodebench-official-20240101-20241101", "score": 0.489, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-V3'; model_name=deepseek-chat. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=48.9405684755%; website rounding=48.9%; divided by 100. deepseek-chat endpoint explicitly labelled DeepSeek-V3, linked to original DeepSeek-V3, released 2024-12-26 before V3-0324 existed. No later endpoint alias resolution used. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "DeepSeek-V3" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 48.940568475452196, "published_display_percent": 48.9, "scale_divisor": 100, "provenance_id": "deepseek-v3/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "deepseek-chat", "upstream_model_repr": "DeepSeek-V3", "upstream_model_style": "DeepSeekCode", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "llama-3-1-405b", "bench": "livecodebench-official-20240101-20241101", "score": 0.373, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'LLama3.1-405b-Ins-FP8'; model_name=meta-llama/Meta-Llama-3.1-405B-Instruct-FP8. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=37.2868217054%; website rounding=37.3%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Source checkpoint is specifically FP8, retained in settings and note. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "LLama3.1-405b-Ins-FP8" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 37.286821705426355, "published_display_percent": 37.3, "scale_divisor": 100, "provenance_id": "llama-3-1-405b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8", "upstream_model_repr": "LLama3.1-405b-Ins-FP8", "upstream_model_style": "LLaMa3", "reasoning_effort": null, "weight_precision": "FP8", "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "llama-3-1-70b", "bench": "livecodebench-official-20240101-20241101", "score": 0.263, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'LLama3.1-70b-Ins'; model_name=meta-llama/Meta-Llama-3.1-70B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=26.2532299742%; website rounding=26.3%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "LLama3.1-70b-Ins" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 26.253229974160206, "published_display_percent": 26.3, "scale_divisor": 100, "provenance_id": "llama-3-1-70b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "meta-llama/Meta-Llama-3.1-70B-Instruct", "upstream_model_repr": "LLama3.1-70b-Ins", "upstream_model_style": "LLaMa3", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "llama-3-1-8b", "bench": "livecodebench-official-20240101-20241101", "score": 0.161, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'LLama3.1-8b-Ins'; model_name=meta-llama/Meta-Llama-3.1-8B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=16.149870801%; website rounding=16.1%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "LLama3.1-8b-Ins" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 16.149870801033593, "published_display_percent": 16.1, "scale_divisor": 100, "provenance_id": "llama-3-1-8b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "meta-llama/Meta-Llama-3.1-8B-Instruct", "upstream_model_repr": "LLama3.1-8b-Ins", "upstream_model_style": "LLaMa3", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "llama-3-3-70b", "bench": "livecodebench-official-20240101-20241101", "score": 0.318, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'LLama3.3-70b-Ins'; model_name=meta-llama/Llama-3.3-70B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=31.7829457364%; website rounding=31.8%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Upstream link mistakenly points to Llama 3.1; model_name and model_repr both explicitly identify Llama 3.3 and take precedence. Raw metadata retained. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "LLama3.3-70b-Ins" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 31.782945736434108, "published_display_percent": 31.8, "scale_divisor": 100, "provenance_id": "llama-3-3-70b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "meta-llama/Llama-3.3-70B-Instruct", "upstream_model_repr": "LLama3.3-70b-Ins", "upstream_model_style": "LLaMa3", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "qwen2-5-32b", "bench": "livecodebench-official-20240101-20241101", "score": 0.401, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Ins-32B'; model_name=Qwen/Qwen2.5-32B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=40.0775193798%; website rounding=40.1%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Qwen2.5-Ins-32B" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 40.07751937984496, "published_display_percent": 40.1, "scale_divisor": 100, "provenance_id": "qwen2-5-32b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "Qwen/Qwen2.5-32B-Instruct", "upstream_model_repr": "Qwen2.5-Ins-32B", "upstream_model_style": "CodeQwen15", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "qwen2-5-72b", "bench": "livecodebench-official-20240101-20241101", "score": 0.41700000000000004, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Ins-72B'; model_name=Qwen/Qwen2.5-72B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=41.6795865633%; website rounding=41.7%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Qwen2.5-Ins-72B" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 41.679586563307495, "published_display_percent": 41.7, "scale_divisor": 100, "provenance_id": "qwen2-5-72b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "Qwen/Qwen2.5-72B-Instruct", "upstream_model_repr": "Qwen2.5-Ins-72B", "upstream_model_style": "CodeQwen15", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "qwen2-5-7b", "bench": "livecodebench-official-20240101-20241101", "score": 0.253, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Ins-7B'; model_name=Qwen/Qwen2.5-7B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=25.322997416%; website rounding=25.3%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Qwen2.5-Ins-7B" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 25.322997416020673, "published_display_percent": 25.3, "scale_divisor": 100, "provenance_id": "qwen2-5-7b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "Qwen/Qwen2.5-7B-Instruct", "upstream_model_repr": "Qwen2.5-Ins-7B", "upstream_model_style": "CodeQwen15", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "qwen2-5-coder-32b", "bench": "livecodebench-official-20240101-20241101", "score": 0.41600000000000004, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Coder-Ins-32B'; model_name=Qwen/Qwen2.5-Coder-32B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=41.6279069767%; website rounding=41.6%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Qwen2.5-Coder-Ins-32B" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 41.627906976744185, "published_display_percent": 41.6, "scale_divisor": 100, "provenance_id": "qwen2-5-coder-32b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "Qwen/Qwen2.5-Coder-32B-Instruct", "upstream_model_repr": "Qwen2.5-Coder-Ins-32B", "upstream_model_style": "CodeQwen15", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "qwen2-5-coder-7b", "bench": "livecodebench-official-20240101-20241101", "score": 0.262, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Qwen2.5-Coder-Ins-7B'; model_name=Qwen/Qwen2.5-Coder-7B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=26.2273901809%; website rounding=26.2%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Qwen2.5-Coder-Ins-7B" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 26.227390180878555, "published_display_percent": 26.2, "scale_divisor": 100, "provenance_id": "qwen2-5-coder-7b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "Qwen/Qwen2.5-Coder-7B-Instruct", "upstream_model_repr": "Qwen2.5-Coder-Ins-7B", "upstream_model_style": "CodeQwen15", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "o3", "bench": "livecodebench-official-20240801-20250501", "score": 0.758, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "source_id": "livecodebench-website-20250801", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'O3 (High)'; model_name=o3__high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=75.7709251101%; website rounding=75.8%; divided by 100. Exact published endpoint/config retained (o3__high); source does not disclose a dated API snapshot. High effort selected a priori, not by maximising the score. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "O3 (High)" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 75.77092511013215, "published_display_percent": 75.8, "scale_divisor": 100, "provenance_id": "o3/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "o3__high", "upstream_model_repr": "O3 (High)", "upstream_model_style": "OpenAIReason", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "o4-mini", "bench": "livecodebench-official-20240801-20250501", "score": 0.802, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "source_id": "livecodebench-website-20250801", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'O4-Mini (High)'; model_name=o4-mini__high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=80.1762114537%; website rounding=80.2%; divided by 100. Exact published endpoint/config retained (o4-mini__high); source does not disclose a dated API snapshot. High effort selected a priori, not by maximising the score. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "O4-Mini (High)" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 80.1762114537445, "published_display_percent": 80.2, "scale_divisor": 100, "provenance_id": "o4-mini/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "o4-mini__high", "upstream_model_repr": "O4-Mini (High)", "upstream_model_style": "OpenAIReason", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "qwen3-235b-a22b", "bench": "livecodebench-official-20240801-20250501", "score": 0.659, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "source_id": "livecodebench-website-20250801", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Qwen3-235B-A22B'; model_name=Qwen/Qwen3-235B-A22B. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=65.859030837%; website rounding=65.9%; divided by 100. Exact upstream model_name matched to original catalogue; instruct/base and parameter size preserved. April 2025 hybrid checkpoint; thinking mode/token budget is not stated by the website JSON. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Qwen3-235B-A22B" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 65.85903083700441, "published_display_percent": 65.9, "scale_divisor": 100, "provenance_id": "qwen3-235b-a22b/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "Qwen/Qwen3-235B-A22B", "upstream_model_repr": "Qwen3-235B-A22B", "upstream_model_style": "CodeQwenInstruct", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "deepseek-coder-1-3b-instruct", "bench": "livecodebench-official-20240101-20241101", "score": 0.065, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'DSCoder-1.3b-Ins'; model_name=deepseek-ai/deepseek-coder-1.3b-instruct; link https://huggingface.co/deepseek-ai/deepseek-coder-1.3b-instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=6.4857881137%; website rounding=6.5%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "deepseek-coder-6-7b-instruct", "bench": "livecodebench-official-20240101-20241101", "score": 0.179, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'DSCoder-6.7b-Ins'; model_name=deepseek-ai/deepseek-coder-6.7b-instruct; link https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=17.8811369509%; website rounding=17.9%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "deepseek-coder-33b-instruct", "bench": "livecodebench-official-20240101-20241101", "score": 0.219, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'DSCoder-33b-Ins'; model_name=deepseek-ai/deepseek-coder-33b-instruct; link https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=21.8604651163%; website rounding=21.9%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "deepseek-coder-v2-lite-instruct", "bench": "livecodebench-official-20240101-20241101", "score": 0.206, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-Coder-V2-Lite-Instruct'; model_name=deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct; link https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=20.5684754522%; website rounding=20.6%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "deepseek-v2-lite-chat", "bench": "livecodebench-official-20240101-20241101", "score": 0.036, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-V2-Lite-Chat'; model_name=deepseek-ai/DeepSeek-V2-Lite-Chat; link https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=3.5917312661%; website rounding=3.6%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "llama-3-8b", "bench": "livecodebench-official-20240101-20241101", "score": 0.133, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'LLama3-8b-Ins'; model_name=meta-llama/Meta-Llama-3-8B-Instruct; link https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=13.3333333333%; website rounding=13.3%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "llama-3-70b", "bench": "livecodebench-official-20240101-20241101", "score": 0.243, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'LLama3-70b-Ins'; model_name=meta-llama/Meta-Llama-3-70B-Instruct; link https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=24.3152454780%; website rounding=24.3%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "qwen2-72b", "bench": "livecodebench-official-20240101-20241101", "score": 0.253, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Qwen2-Ins-72B'; model_name=Qwen/Qwen2-72B-Instruct; link https://huggingface.co/Qwen/Qwen2-72B-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=25.3488372093%; website rounding=25.3%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "yi-coder-9b-chat", "bench": "livecodebench-official-20240101-20241101", "score": 0.235, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Yi-Coder-9B-Chat'; model_name=01-ai/Yi-Coder-9B-Chat; link https://huggingface.co/01-ai/Yi-Coder-9B-Chat. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=23.5400516796%; website rounding=23.5%; divided by 100. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance." }, { "model": "deepseek-coder-v2", "bench": "livecodebench-official-20240101-20241101", "score": 0.344, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'DeepSeek-Coder-V2-Instruct'; model_name=deepseek-ai/DeepSeek-Coder-V2-Instruct. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=34.4186046512%; website rounding=34.4%; divided by 100. Exact upstream model_name deepseek-ai/DeepSeek-Coder-V2-Instruct (the original June 2024 release, catalogued as DeepSeek-Coder-V2 Instruct (0614)); the separate -0724 entry is not merged. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "DeepSeek-Coder-V2-Instruct" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 34.41860465116279, "published_display_percent": 34.4, "scale_divisor": 100, "provenance_id": "deepseek-coder-v2/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "deepseek-ai/DeepSeek-Coder-V2-Instruct", "upstream_model_repr": "DeepSeek-Coder-V2-Instruct", "upstream_model_style": "DeepSeekCode", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "gpt-4-0613", "bench": "livecodebench-official-20240101-20241101", "score": 0.325, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'GPT-4-0613'; model_name=gpt-4-0613. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=32.480620155%; website rounding=32.5%; divided by 100. Exact dated API snapshot gpt-4-0613. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "GPT-4-0613" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 32.48062015503876, "published_display_percent": 32.5, "scale_divisor": 100, "provenance_id": "gpt-4-0613/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gpt-4-0613", "upstream_model_repr": "GPT-4-0613", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "gpt-4-1106-preview", "bench": "livecodebench-official-20240101-20241101", "score": 0.357, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'GPT-4-Turbo-1106'; model_name=gpt-4-1106-preview. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=35.7364341085%; website rounding=35.7%; divided by 100. Exact dated API snapshot gpt-4-1106-preview. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "GPT-4-Turbo-1106" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 35.736434108527135, "published_display_percent": 35.7, "scale_divisor": 100, "provenance_id": "gpt-4-1106-preview/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gpt-4-1106-preview", "upstream_model_repr": "GPT-4-Turbo-1106", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "gpt-4o-2024-05-13", "bench": "livecodebench-official-20240101-20241101", "score": 0.401, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'GPT-4O-2024-05-13'; model_name=gpt-4o-2024-05-13. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=40.1291989664%; website rounding=40.1%; divided by 100. Exact dated API snapshot gpt-4o-2024-05-13. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "GPT-4O-2024-05-13" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 40.12919896640827, "published_display_percent": 40.1, "scale_divisor": 100, "provenance_id": "gpt-4o-2024-05-13/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gpt-4o-2024-05-13", "upstream_model_repr": "GPT-4O-2024-05-13", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "o1-preview", "bench": "livecodebench-official-20240101-20241101", "score": 0.525, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'O1-Preview-2024-09-12 (N=1)'; model_name=o1-preview-2024-09-12. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=52.4547803618%; website rounding=52.5%; divided by 100. Exact dated API snapshot o1-preview-2024-09-12 (the only o1-preview snapshot); label '(N=1)' retained. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "O1-Preview-2024-09-12 (N=1)" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 52.45478036175711, "published_display_percent": 52.5, "scale_divisor": 100, "provenance_id": "o1-preview/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "o1-preview-2024-09-12", "upstream_model_repr": "O1-Preview-2024-09-12 (N=1)", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "o1-mini", "bench": "livecodebench-official-20240101-20241101", "score": 0.672, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'O1-Mini-2024-09-12 (N=1)'; model_name=o1-mini-2024-09-12. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=67.1834625323%; website rounding=67.2%; divided by 100. Exact dated API snapshot o1-mini-2024-09-12 (the only o1-mini snapshot); label '(N=1)' retained. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "O1-Mini-2024-09-12 (N=1)" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 67.18346253229974, "published_display_percent": 67.2, "scale_divisor": 100, "provenance_id": "o1-mini/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "o1-mini-2024-09-12", "upstream_model_repr": "O1-Mini-2024-09-12 (N=1)", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "claude-3-opus-20240229", "bench": "livecodebench-official-20240101-20241101", "score": 0.319, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Claude-3-Opus'; model_name=claude-3-opus-20240229. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=31.9379844961%; website rounding=31.9%; divided by 100. Exact dated API snapshot claude-3-opus-20240229. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Claude-3-Opus" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 31.93798449612403, "published_display_percent": 31.9, "scale_divisor": 100, "provenance_id": "claude-3-opus-20240229/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "claude-3-opus-20240229", "upstream_model_repr": "Claude-3-Opus", "upstream_model_style": "AnthropicMessage", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "claude-3-sonnet-20240229", "bench": "livecodebench-official-20240101-20241101", "score": 0.25, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Claude-3-Sonnet'; model_name=claude-3-sonnet-20240229. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=25.0129198966%; website rounding=25%; divided by 100. Exact dated API snapshot claude-3-sonnet-20240229. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Claude-3-Sonnet" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 25.012919896640827, "published_display_percent": 25, "scale_divisor": 100, "provenance_id": "claude-3-sonnet-20240229/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "claude-3-sonnet-20240229", "upstream_model_repr": "Claude-3-Sonnet", "upstream_model_style": "AnthropicMessage", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "claude-3-5-sonnet-20240620", "bench": "livecodebench-official-20240101-20241101", "score": 0.426, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Claude-3.5-Sonnet-20240620'; model_name=claude-3-5-sonnet-20240620. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=42.6098191214%; website rounding=42.6%; divided by 100. Exact dated API snapshot claude-3-5-sonnet-20240620. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Claude-3.5-Sonnet-20240620" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 42.60981912144703, "published_display_percent": 42.6, "scale_divisor": 100, "provenance_id": "claude-3-5-sonnet-20240620/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "claude-3-5-sonnet-20240620", "upstream_model_repr": "Claude-3.5-Sonnet-20240620", "upstream_model_style": "AnthropicMessage", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "gemini-1-5-pro-002", "bench": "livecodebench-official-20240101-20241101", "score": 0.364, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Gemini-Pro-1.5-002'; model_name=gemini-1.5-pro-002. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=36.4341085271%; website rounding=36.4%; divided by 100. Exact dated API version gemini-1.5-pro-002. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Gemini-Pro-1.5-002" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 36.434108527131784, "published_display_percent": 36.4, "scale_divisor": 100, "provenance_id": "gemini-1-5-pro-002/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-1.5-pro-002", "upstream_model_repr": "Gemini-Pro-1.5-002", "upstream_model_style": "Google", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "gemini-1-5-flash-002", "bench": "livecodebench-official-20240101-20241101", "score": 0.344, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'Gemini-Flash-1.5-002'; model_name=gemini-1.5-flash-002. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=34.3669250646%; website rounding=34.4%; divided by 100. Exact dated API version gemini-1.5-flash-002. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "Gemini-Flash-1.5-002" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 34.366925064599485, "published_display_percent": 34.4, "scale_divisor": 100, "provenance_id": "gemini-1-5-flash-002/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-1.5-flash-002", "upstream_model_repr": "Gemini-Flash-1.5-002", "upstream_model_style": "Google", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "o1-2024-12-17", "bench": "livecodebench-official-20240101-20241101", "score": 0.724, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'O1-2024-12-17 (N=1) (Tentative)'; model_name=o1-2024-12-17. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=72.3514211886%; website rounding=72.4%; divided by 100. Exact dated API snapshot o1-2024-12-17; the source itself labels this entry '(N=1) (Tentative)', retained here; reasoning effort not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "O1-2024-12-17 (N=1) (Tentative)" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 72.3514211886305, "published_display_percent": 72.4, "scale_divisor": 100, "provenance_id": "o1-2024-12-17/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "o1-2024-12-17", "upstream_model_repr": "O1-2024-12-17 (N=1) (Tentative)", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20241226" }, { "model": "gpt-4-turbo-2024-04-09", "bench": "livecodebench-official-20240801-20250501", "score": 0.287, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'GPT-4-Turbo-2024-04-09'; model_name=gpt-4-turbo-2024-04-09. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=28.7004405286%; website rounding=28.7%; divided by 100. Exact dated API snapshot gpt-4-turbo-2024-04-09. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "GPT-4-Turbo-2024-04-09" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 28.70044052863436, "published_display_percent": 28.7, "scale_divisor": 100, "provenance_id": "gpt-4-turbo-2024-04-09/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gpt-4-turbo-2024-04-09", "upstream_model_repr": "GPT-4-Turbo-2024-04-09", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "gpt-4o-2024-08-06", "bench": "livecodebench-official-20240801-20250501", "score": 0.295, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'GPT-4O-2024-08-06'; model_name=gpt-4o-2024-08-06. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=29.5374449339%; website rounding=29.5%; divided by 100. Exact dated API snapshot gpt-4o-2024-08-06. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "GPT-4O-2024-08-06" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 29.537444933920703, "published_display_percent": 29.5, "scale_divisor": 100, "provenance_id": "gpt-4o-2024-08-06/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gpt-4o-2024-08-06", "upstream_model_repr": "GPT-4O-2024-08-06", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "gpt-4o-mini-2024-07-18", "bench": "livecodebench-official-20240801-20250501", "score": 0.275, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'GPT-4O-mini-2024-07-18'; model_name=gpt-4o-mini-2024-07-18. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=27.4669603524%; website rounding=27.5%; divided by 100. Exact dated API snapshot gpt-4o-mini-2024-07-18. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "GPT-4O-mini-2024-07-18" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 27.46696035242291, "published_display_percent": 27.5, "scale_divisor": 100, "provenance_id": "gpt-4o-mini-2024-07-18/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gpt-4o-mini-2024-07-18", "upstream_model_repr": "GPT-4O-mini-2024-07-18", "upstream_model_style": "OpenAIChat", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "o3-mini", "bench": "livecodebench-official-20240801-20250501", "score": 0.674, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'O3-Mini-2025-01-31 (High)'; model_name=o3-mini-2025-01-31__high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=67.4008810573%; website rounding=67.4%; divided by 100. Exact dated API snapshot o3-mini-2025-01-31 with published config o3-mini-2025-01-31__high. High effort selected a priori (Low/Med entries of the same snapshot not imported), not by maximising the score. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "O3-Mini-2025-01-31 (High)" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 67.40088105726872, "published_display_percent": 67.4, "scale_divisor": 100, "provenance_id": "o3-mini/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "o3-mini-2025-01-31__high", "upstream_model_repr": "O3-Mini-2025-01-31 (High)", "upstream_model_style": "OpenAIReason", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "claude-3-5-sonnet-20241022", "bench": "livecodebench-official-20240801-20250501", "score": 0.364, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Claude-3.5-Sonnet-20241022'; model_name=claude-3-5-sonnet-20241022. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=36.3876651982%; website rounding=36.4%; divided by 100. Exact dated API snapshot claude-3-5-sonnet-20241022. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Claude-3.5-Sonnet-20241022" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 36.38766519823788, "published_display_percent": 36.4, "scale_divisor": 100, "provenance_id": "claude-3-5-sonnet-20241022/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "claude-3-5-sonnet-20241022", "upstream_model_repr": "Claude-3.5-Sonnet-20241022", "upstream_model_style": "Claude3", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "claude-3-haiku-20240307", "bench": "livecodebench-official-20240801-20250501", "score": 0.202, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Claude-3-Haiku'; model_name=claude-3-haiku-20240307. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=20.2202643172%; website rounding=20.2%; divided by 100. Exact dated API snapshot claude-3-haiku-20240307. Also listed in the 2024-12-26 release; one window per model, the newer release kept. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Claude-3-Haiku" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 20.220264317180618, "published_display_percent": 20.2, "scale_divisor": 100, "provenance_id": "claude-3-haiku-20240307/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "claude-3-haiku-20240307", "upstream_model_repr": "Claude-3-Haiku", "upstream_model_style": "Claude3", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "claude-sonnet-4-20250514", "bench": "livecodebench-official-20240801-20250501", "score": 0.559, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Claude-Sonnet-4 (Thinking)'; model_name=claude-sonnet-4-20250514. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=55.9471365639%; website rounding=55.9%; divided by 100. Exact dated API snapshot claude-sonnet-4-20250514 with extended thinking (model_style Claude3Thinking); the no-thinking entry of the same snapshot is not imported. Thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Claude-Sonnet-4 (Thinking)" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 55.947136563876654, "published_display_percent": 55.9, "scale_divisor": 100, "provenance_id": "claude-sonnet-4-20250514/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "claude-sonnet-4-20250514", "upstream_model_repr": "Claude-Sonnet-4 (Thinking)", "upstream_model_style": "Claude3Thinking", "reasoning_effort": "thinking", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "claude-opus-4-20250514", "bench": "livecodebench-official-20240801-20250501", "score": 0.566, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Claude-Opus-4 (Thinking)'; model_name=claude-opus-4-20250514. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=56.6079295154%; website rounding=56.6%; divided by 100. Exact dated API snapshot claude-opus-4-20250514 with extended thinking (model_style Claude3Thinking); the no-thinking entry of the same snapshot is not imported. Thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Claude-Opus-4 (Thinking)" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 56.607929515418505, "published_display_percent": 56.6, "scale_divisor": 100, "provenance_id": "claude-opus-4-20250514/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "claude-opus-4-20250514", "upstream_model_repr": "Claude-Opus-4 (Thinking)", "upstream_model_style": "Claude3Thinking", "reasoning_effort": "thinking", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "gemini-2-5-pro-preview-06-05", "bench": "livecodebench-official-20240801-20250501", "score": 0.736, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Pro-06-05'; model_name=gemini-2.5-pro-preview-06-05. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=73.5682819383%; website rounding=73.6%; divided by 100. Exact dated preview API id gemini-2.5-pro-preview-06-05 (catalogued separately from GA gemini-2.5-pro). Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Gemini-2.5-Pro-06-05" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 73.568281938326, "published_display_percent": 73.6, "scale_divisor": 100, "provenance_id": "gemini-2-5-pro-preview-06-05/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-2.5-pro-preview-06-05", "upstream_model_repr": "Gemini-2.5-Pro-06-05", "upstream_model_style": "GeminiThinking", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "gemini-2-5-pro-preview-05-06", "bench": "livecodebench-official-20240801-20250501", "score": 0.718, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Pro-05-06'; model_name=gemini-2.5-pro-preview-05-06. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=71.8061674009%; website rounding=71.8%; divided by 100. Exact dated preview API id gemini-2.5-pro-preview-05-06. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Gemini-2.5-Pro-05-06" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 71.80616740088105, "published_display_percent": 71.8, "scale_divisor": 100, "provenance_id": "gemini-2-5-pro-preview-05-06/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-2.5-pro-preview-05-06", "upstream_model_repr": "Gemini-2.5-Pro-05-06", "upstream_model_style": "GeminiThinking", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "gemini-2-5-flash-preview-04-17", "bench": "livecodebench-official-20240801-20250501", "score": 0.606, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Flash-04-17'; model_name=gemini-2.5-flash-preview-04-17. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=60.5726872247%; website rounding=60.6%; divided by 100. Exact dated preview API id gemini-2.5-flash-preview-04-17; thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Gemini-2.5-Flash-04-17" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 60.5726872246696, "published_display_percent": 60.6, "scale_divisor": 100, "provenance_id": "gemini-2-5-flash-preview-04-17/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-2.5-flash-preview-04-17", "upstream_model_repr": "Gemini-2.5-Flash-04-17", "upstream_model_style": "GeminiThinking", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "gemini-2-5-flash-preview-05-20", "bench": "livecodebench-official-20240801-20250501", "score": 0.619, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Gemini-2.5-Flash-05-20'; model_name=gemini-2.5-flash-preview-05-20. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=61.8942731278%; website rounding=61.9%; divided by 100. Exact dated preview API id gemini-2.5-flash-preview-05-20; thinking budget not stated. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Gemini-2.5-Flash-05-20" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 61.8942731277533, "published_display_percent": 61.9, "scale_divisor": 100, "provenance_id": "gemini-2-5-flash-preview-05-20/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-2.5-flash-preview-05-20", "upstream_model_repr": "Gemini-2.5-Flash-05-20", "upstream_model_style": "GeminiThinking", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "grok-3-mini-beta", "bench": "livecodebench-official-20240801-20250501", "score": 0.667, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release; entry 'Grok-3-Mini (High)'; model_name=grok-3-mini-beta_high. Code generation, 454 problems; inclusive UTC window 2024-08-01T00:00:00Z to 2025-05-01T00:00:00Z. Mean per-problem pass@1=66.7400881057%; website rounding=66.7%; divided by 100. Exact API id grok-3-mini-beta with published config grok-3-mini-beta_high; high effort selected a priori. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/performances_generation.json", "model_selector": { "model": "Grok-3-Mini (High)" }, "date_filter": { "gte": 1722470400000, "lte": 1746057600000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 454, "raw_mean_percent": 66.74008810572687, "published_display_percent": 66.7, "scale_divisor": 100, "provenance_id": "grok-3-mini-beta/livecodebench-official-20240801-20250501", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "grok-3-mini-beta_high", "upstream_model_repr": "Grok-3-Mini (High)", "upstream_model_style": "Grok", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json", "source_id": "livecodebench-website-20250801" }, { "model": "minicpm3-4b", "bench": "livecodebench-official-20240101-20241101", "score": 0.215, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "source_id": "livecodebench-website-20241226", "by": "independent", "date": "2024-12-26", "note": "LiveCodeBench author-maintained website release; entry 'MiniCPM3-4B'; model_name=openbmb/MiniCPM3-4B. Code generation, 387 problems; inclusive UTC window 2024-01-01T00:00:00Z to 2024-11-01T00:00:00Z. Mean per-problem pass@1=21.5245478036%; website rounding=21.5%; divided by 100. Exact upstream model_name matched to original catalogue (catalogue minicpm3-4b = openbmb/MiniCPM3-4B). Same window and the same 387 problem IDs as the existing rows; method reproduces deepseek-v3's 48.9405684755%. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings. Date is publication date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Added in pass 12 (held in pass 10 only because the extract files were missing).", "evidence": { "kind": "published-per-problem-json", "commit": "aa37a9f8317713b146ff5ada8ab8b2513bb20b70", "file": "src/mocks/performances_generation.json", "model_selector": { "model": "MiniCPM3-4B" }, "date_filter": { "gte": 1704067200000, "lte": 1730419200000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 387, "raw_mean_percent": 21.524547803617573, "published_display_percent": 21.5, "scale_divisor": 100, "provenance_id": "minicpm3-4b/livecodebench-official-20240101-20241101", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "openbmb/MiniCPM3-4B", "upstream_model_repr": "MiniCPM3-4B", "upstream_model_style": "Misc", "reasoning_effort": null, "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/aa37a9f8317713b146ff5ada8ab8b2513bb20b70/src/mocks/performances_generation.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "gemini-2-5-pro-exp-03-25", "bench": "livecodebench-official-20231101-20250201", "score": 0.809, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "source_id": "livecodebench-website-20250801", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'Gemini-2.5-Pro'; model_name=gemini-2.5-pro-exp-03-25. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=80.9220985692%; website rounding=80.9%; divided by 100. Identity: upstream model_name 'gemini-2.5-pro-exp-03-25' equals the catalogued dated experimental id. One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/v5.json", "model_selector": { "model": "Gemini-2.5-Pro" }, "date_filter": { "gte": 1698796800000, "lte": 1738368000000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 629, "raw_mean_percent": 80.9220985691574, "published_display_percent": 80.9, "scale_divisor": 100, "provenance_id": "gemini-2-5-pro-exp-03-25/livecodebench-official-20231101-20250201", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-2.5-pro-exp-03-25", "upstream_model_repr": "Gemini-2.5-Pro", "upstream_model_style": "GeminiThinking", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "qwq-32b-preview", "bench": "livecodebench-official-20231101-20250201", "score": 0.558, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "source_id": "livecodebench-website-20250801", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'QwQ-32B-Preview'; model_name=Qwen/QwQ-32B-Preview. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=55.8028616852%; website rounding=55.8%; divided by 100. Identity: upstream model_name 'Qwen/QwQ-32B-Preview' equals the catalogued repo. One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/v5.json", "model_selector": { "model": "QwQ-32B-Preview" }, "date_filter": { "gte": 1698796800000, "lte": 1738368000000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 629, "raw_mean_percent": 55.80286168521462, "published_display_percent": 55.8, "scale_divisor": 100, "provenance_id": "qwq-32b-preview/livecodebench-official-20231101-20250201", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "Qwen/QwQ-32B-Preview", "upstream_model_repr": "QwQ-32B-Preview", "upstream_model_style": "QwQ", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "gemini-2-0-flash-exp", "bench": "livecodebench-official-20231101-20250201", "score": 0.379, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "source_id": "livecodebench-website-20250801", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'Gemini-Flash-2.0-Exp'; model_name=gemini-2.0-flash-exp. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=37.8537360890%; website rounding=37.9%; divided by 100. Identity: upstream model_name 'gemini-2.0-flash-exp' equals the catalogued experimental id. One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/v5.json", "model_selector": { "model": "Gemini-Flash-2.0-Exp" }, "date_filter": { "gte": 1698796800000, "lte": 1738368000000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 629, "raw_mean_percent": 37.85373608903021, "published_display_percent": 37.9, "scale_divisor": 100, "provenance_id": "gemini-2-0-flash-exp/livecodebench-official-20231101-20250201", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "gemini-2.0-flash-exp", "upstream_model_repr": "Gemini-Flash-2.0-Exp", "upstream_model_style": "Gemini", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" }, { "model": "llama-3-1-nemotron-ultra-253b", "bench": "livecodebench-official-20231101-20250201", "score": 0.749, "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "source_id": "livecodebench-website-20250801", "by": "independent", "date": "2025-08-01", "note": "LiveCodeBench author-maintained website release_v5 data (build/v5.json at f186ab5, the file behind leaderboard_v5.html); entry 'Llama-3_1-Nemotron-Ultra-253B-v1'; model_name=nvidia/Llama-3_1-Nemotron-Ultra-253B-v1. Code generation, 629 problems; inclusive UTC window 2023-11-01T00:00:00Z to 2025-02-01T00:00:00Z = the page's default slider bounds (startSlider 30 -> date_marks[6], endSlider 100 -> date_marks[21]; filter r.date >= start && r.date <= end). Mean per-problem pass@1=74.8807631161%; website rounding=74.9%; divided by 100. Identity: upstream model_name 'nvidia/Llama-3_1-Nemotron-Ultra-253B-v1' equals the catalogued repo; no vendor submission PR for it exists in LiveCodeBench/submissions (the frozen 2026-09 review withheld it because its own-run origin was not established; imported under the 2026-10-11 owner decision). One LiveCodeBench window per model: none of these models has another LCB row. Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Date is the pinned commit date, not evaluation-run date. CC BY-SA 4.0; attribution and adaptation notice retained in source/licensing/provenance. Held in pass 8 (file outside the ledger match; one-window policy); imported under the 2026-10-11 owner decision.", "evidence": { "kind": "published-per-problem-json", "commit": "f186ab5041d6b768733b8ae8145151c867942134", "file": "build/v5.json", "model_selector": { "model": "Llama-3_1-Nemotron-Ultra-253B-v1" }, "date_filter": { "gte": 1698796800000, "lte": 1738368000000 }, "field": "pass@1", "aggregation": "Number(mean(values).toFixed(1)) / 100", "problem_count": 629, "raw_mean_percent": 74.88076311605724, "published_display_percent": 74.9, "scale_divisor": 100, "provenance_id": "llama-3-1-nemotron-ultra-253b/livecodebench-official-20231101-20250201", "provenance_file": "/data/lcb-sharealike-provenance.json" }, "settings": { "metric": "pass@1", "scenario": "codegeneration", "upstream_model_name": "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1", "upstream_model_repr": "Llama-3_1-Nemotron-Ultra-253B-v1", "upstream_model_style": "DeepSeekR1", "reasoning_effort": "high", "weight_precision": null, "temperature": null, "top_p": null, "samples_per_problem": null, "max_output_tokens": null, "evaluation_run_date": null, "exact_harness_commit": null, "weight_digest": null, "unknown_settings_note": "Per-run decoding, sample counts, token budgets, run date and weight digests are not released in these website records. Harness defaults are not asserted as actual run settings." }, "licensing": { "license": "CC-BY-SA-4.0", "license_url": "https://creativecommons.org/licenses/by-sa/4.0/", "adapter_license": "CC-BY-SA-4.0", "attribution": "LiveCodeBench website evaluation results by the LiveCodeBench authors (Naman Jain, King Han, Alex Gu, Wen-Ding Li, Fanjia Yan, Tianjun Zhang, Sida Wang, Armando Solar-Lezama, Koushik Sen, Ion Stoica). CC BY-SA 4.0. Adapted by whatmodel.sh: selected models and explicit date windows, reproduced the website arithmetic mean and one-decimal percentage rounding, mapped catalogue IDs, converted percentages to fractions, and added provenance. No endorsement.", "source": "https://github.com/LiveCodeBench/livecodebench.github.io/blob/f186ab5041d6b768733b8ae8145151c867942134/build/v5.json", "disclaimer": "Licensed material is provided as-is and as-available, without warranties; see CC BY-SA 4.0 section 5 for the complete disclaimer and liability limitation.", "notice_file": "/licenses/lcb-sharealike-NOTICE.txt", "adapted_data_file": "/data/lcb-sharealike.json" }, "data_license": "CC-BY-SA-4.0", "licensed_artifact": "/data/lcb-sharealike.json" } ], "models": [ "deepseek-coder-1-3b-instruct", "deepseek-coder-6-7b-instruct", "deepseek-coder-33b-instruct", "deepseek-coder-v2-lite-instruct", "deepseek-v2-lite-chat", "llama-3-8b", "llama-3-70b", "qwen2-72b", "yi-coder-9b-chat", "deepseek-coder-v2", "gpt-4-0613", "gpt-4-1106-preview", "gpt-4o-2024-05-13", "o1-preview", "o1-mini", "claude-3-opus-20240229", "claude-3-sonnet-20240229", "claude-3-5-sonnet-20240620", "gemini-1-5-pro-002", "gemini-1-5-flash-002", "o1-2024-12-17", "gpt-4-turbo-2024-04-09", "gpt-4o-2024-08-06", "gpt-4o-mini-2024-07-18", "o3-mini", "claude-3-5-sonnet-20241022", "claude-3-haiku-20240307", "claude-sonnet-4-20250514", "claude-opus-4-20250514", "gemini-2-5-pro-preview-06-05", "gemini-2-5-pro-preview-05-06", "gemini-2-5-flash-preview-04-17", "gemini-2-5-flash-preview-05-20", "grok-3-mini-beta", "minicpm3-4b", "gemini-2-5-pro-exp-03-25", "qwq-32b-preview", "gemini-2-0-flash-exp", "llama-3-1-nemotron-ultra-253b" ] }