{
"table": [],
"findings": [
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q8_0",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6523,
"retention": 1.0004601226993866,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L159",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 159,
"raw_line": "| 7.43 | 65.23 | 8.50 | 8B | Q8_0 | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q6_K",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6506000000000001,
"retention": 0.9978527607361963,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L163",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 163,
"raw_line": "| 5.73 | 65.06 | 6.56 | 8B | Q6_K | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q5_K_M",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.649,
"retention": 0.995398773006135,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L164",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 164,
"raw_line": "| 5.00 | 64.90 | 5.67 | 8B | Q5_K_M | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q5_K_S",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6487999999999999,
"retention": 0.9950920245398771,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L165",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 165,
"raw_line": "| 4.87 | 64.88 | 5.50 | 8B | Q5_K_S | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q4_K_M",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6464,
"retention": 0.9914110429447852,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L167",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 167,
"raw_line": "| 4.30 | 64.64 | 4.82 | 8B | Q4_K_M | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q4_K_S",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6463,
"retention": 0.9912576687116564,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L168",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 168,
"raw_line": "| 4.09 | 64.63 | 4.54 | 8B | Q4_K_S | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ4_NL",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6433,
"retention": 0.9866564417177913,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L169",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 169,
"raw_line": "| 4.07 | 64.33 | 4.52 | 8B | IQ4_NL | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ4_XS",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6439,
"retention": 0.9875766871165644,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L170",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 170,
"raw_line": "| 3.87 | 64.39 | 4.28 | 8B | IQ4_XS | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q3_K_L",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6285000000000001,
"retention": 0.963957055214724,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L172",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 172,
"raw_line": "| 3.81 | 62.85 | 4.08 | 8B | Q3_K_L | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q3_K_M",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6289,
"retention": 0.9645705521472393,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L173",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 173,
"raw_line": "| 3.53 | 62.89 | 3.79 | 8B | Q3_K_M | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ3_M",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6255,
"retention": 0.9593558282208587,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L176",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 176,
"raw_line": "| 3.31 | 62.55 | 3.50 | 8B | IQ3_M | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ3_S",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6213000000000001,
"retention": 0.952914110429448,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L178",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 178,
"raw_line": "| 3.21 | 62.13 | 3.46 | 8B | IQ3_S | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q3_K_S",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.5914,
"retention": 0.9070552147239264,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L179",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 179,
"raw_line": "| 3.20 | 59.14 | 3.44 | 8B | Q3_K_S | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ3_XS",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6119,
"retention": 0.9384969325153374,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L180",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 180,
"raw_line": "| 3.06 | 61.19 | 3.26 | 8B | IQ3_XS | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ3_XXS",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.6052000000000001,
"retention": 0.9282208588957056,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L181",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 181,
"raw_line": "| 2.83 | 60.52 | 3.04 | 8B | IQ3_XXS | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "Q2_K",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.5589999999999999,
"retention": 0.8573619631901839,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L182",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 182,
"raw_line": "| 2.79 | 55.90 | 2.90 | 8B | Q2_K | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ2_M",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.5756,
"retention": 0.8828220858895706,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L183",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 183,
"raw_line": "| 2.53 | 57.56 | 2.64 | 8B | IQ2_M | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ2_S",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.5398,
"retention": 0.8279141104294477,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L184",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 184,
"raw_line": "| 2.35 | 53.98 | 2.40 | 8B | IQ2_S | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ2_XS",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.49979999999999997,
"retention": 0.766564417177914,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L185",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 185,
"raw_line": "| 2.26 | 49.98 | 2.37 | 8B | IQ2_XS | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ2_XXS",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.435,
"retention": 0.6671779141104294,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L186",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 186,
"raw_line": "| 2.07 | 43.50 | 2.14 | 8B | IQ2_XXS | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ1_M",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.2883,
"retention": 0.44217791411042945,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L187",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 187,
"raw_line": "| 1.85 | 28.83 | 1.84 | 8B | IQ1_M | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "Llama-3-8B-Instruct",
"params_b": 8,
"quant": "IQ1_S",
"metric": "MMLU 5-shot (within-study, nonstandard few-shot selection)",
"full": 0.652,
"quant_score": 0.2647,
"retention": 0.4059815950920245,
"baseline_quant": "fp16",
"source": "https://github.com/matt-c1/llama-3-quant-comparison/blob/f35b7db21acd1f6258239774f3dbc0fd6cfbfbae/README.md#L188",
"source_id": "quant-llama3-cc0",
"evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 188,
"raw_line": "| 1.71 | 26.47 | 1.66 | 8B | IQ1_S | GGUF |"
},
"baseline_evidence": {
"file": "speed-evidence/matt-c1__llama-3-quant-comparison/README.md",
"sha256": "be28dc2fc140ab9b1c2d316ce93e8826b98b903db685cec69894376ff5e0e9fd",
"line": 155,
"raw_line": "| 13.98 | 65.20 | 16.00 | 8B | fp16 | GGUF |"
},
"transformation": "MMLU percentages / 100; retention = quant_score / fp16_score; ratio not rounded",
"note": "Llama 3, not Llama 3.1. Author used first five test questions instead of dev examples, skipped ~20 overlength questions, and did not fix random seed. Relative within-study evidence only; do not compare absolute MMLU to other studies or extrapolate a universal retention floor."
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"params_b": 32,
"quant": "Q8_0",
"metric": "unweighted average of 9 benchmarks",
"full": 0.7778,
"quant_score": 0.7765000000000001,
"retention": 0.9983286191823091,
"baseline_quant": "BF16 (Local Evaluation)",
"source": "https://arxiv.org/html/2505.02390v1#S4.T4",
"source_id": "quant-deepseek-study-v1",
"evidence": {
"file": "speed-evidence/2505.02390v1.html",
"sha256": "9aaa316c52a23180831726885dc743156451f955848ab6c58abca0f277fbf5d7",
"html_row_id": "S4.T4.3.11",
"raw_cells": [
"Average",
"-",
"77.78",
"77.65",
"77.91",
"77.35"
]
},
"transformation": "Published Average percentages / 100; retention = quant_score / local_BF16_score",
"note": "Use local BF16 column, not vendor-reported scores or weighted-average row. Nine benchmarks: AIME2024, MATH500, GPQA, LiveCodeBench, MBPP, MBPP+, MMLU, CMMLU, C-Eval. Ratios above one are retained, not interpreted as proof of an improvement. Full R1/V3 tables use FP8 API baselines and are not silently relabeled full precision."
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"params_b": 32,
"quant": "Q4_K_M",
"metric": "unweighted average of 9 benchmarks",
"full": 0.7778,
"quant_score": 0.7791,
"retention": 1.001671380817691,
"baseline_quant": "BF16 (Local Evaluation)",
"source": "https://arxiv.org/html/2505.02390v1#S4.T4",
"source_id": "quant-deepseek-study-v1",
"evidence": {
"file": "speed-evidence/2505.02390v1.html",
"sha256": "9aaa316c52a23180831726885dc743156451f955848ab6c58abca0f277fbf5d7",
"html_row_id": "S4.T4.3.11",
"raw_cells": [
"Average",
"-",
"77.78",
"77.65",
"77.91",
"77.35"
]
},
"transformation": "Published Average percentages / 100; retention = quant_score / local_BF16_score",
"note": "Use local BF16 column, not vendor-reported scores or weighted-average row. Nine benchmarks: AIME2024, MATH500, GPQA, LiveCodeBench, MBPP, MBPP+, MMLU, CMMLU, C-Eval. Ratios above one are retained, not interpreted as proof of an improvement. Full R1/V3 tables use FP8 API baselines and are not silently relabeled full precision."
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"params_b": 32,
"quant": "Q3_K_M",
"metric": "unweighted average of 9 benchmarks",
"full": 0.7778,
"quant_score": 0.7735,
"retention": 0.9944715865260991,
"baseline_quant": "BF16 (Local Evaluation)",
"source": "https://arxiv.org/html/2505.02390v1#S4.T4",
"source_id": "quant-deepseek-study-v1",
"evidence": {
"file": "speed-evidence/2505.02390v1.html",
"sha256": "9aaa316c52a23180831726885dc743156451f955848ab6c58abca0f277fbf5d7",
"html_row_id": "S4.T4.3.11",
"raw_cells": [
"Average",
"-",
"77.78",
"77.65",
"77.91",
"77.35"
]
},
"transformation": "Published Average percentages / 100; retention = quant_score / local_BF16_score",
"note": "Use local BF16 column, not vendor-reported scores or weighted-average row. Nine benchmarks: AIME2024, MATH500, GPQA, LiveCodeBench, MBPP, MBPP+, MMLU, CMMLU, C-Eval. Ratios above one are retained, not interpreted as proof of an improvement. Full R1/V3 tables use FP8 API baselines and are not silently relabeled full precision."
},
{
"model": "Qwen2.5-14B (base)",
"model_id": null,
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor algorithm not stated)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7971,
"quant_score": 0.7948,
"retention": 0.9971145402082551,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-FP8-dynamic/blob/99a1817b207d2d5684d745e30b3e06473cea1d10/README.md#L102",
"source_id": "quant-redhatai-qwen2-5-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-FP8-dynamic/README.md",
"sha256": "a352fd2c712681d5f8b25eb0a28f86f60b187c74bec6912243bb43f1ca2bbade",
"line": 102,
"raw_line": " | 79.48",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-FP8-dynamic/README.md",
"sha256": "a352fd2c712681d5f8b25eb0a28f86f60b187c74bec6912243bb43f1ca2bbade",
"line": 100,
"raw_line": " | 79.71",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose/header mislabel this checkpoint as INT8; config.json at the pinned revision shows FP8 (float, 8-bit) weights and activations, consistent with the repository name. Quant label taken from config.json. Qwen/Qwen2.5-14B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-14B-FP8-dynamic@99a1817:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-FP8-dynamic",
"artifact_revision": "99a1817b207d2d5684d745e30b3e06473cea1d10",
"published_recovery": "99.7%"
},
{
"model": "Qwen2.5-14B (base)",
"model_id": null,
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor algorithm not stated)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.8446,
"quant_score": 0.8362,
"retention": 0.9900544636514328,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-FP8-dynamic/blob/99a1817b207d2d5684d745e30b3e06473cea1d10/README.md#L122",
"source_id": "quant-redhatai-qwen2-5-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-FP8-dynamic/README.md",
"sha256": "a352fd2c712681d5f8b25eb0a28f86f60b187c74bec6912243bb43f1ca2bbade",
"line": 122,
"raw_line": " | 83.62",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-FP8-dynamic/README.md",
"sha256": "a352fd2c712681d5f8b25eb0a28f86f60b187c74bec6912243bb43f1ca2bbade",
"line": 120,
"raw_line": " | 84.46",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose/header mislabel this checkpoint as INT8; config.json at the pinned revision shows FP8 (float, 8-bit) weights and activations, consistent with the repository name. Quant label taken from config.json. Qwen/Qwen2.5-14B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-14B-FP8-dynamic@99a1817:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-FP8-dynamic",
"artifact_revision": "99a1817b207d2d5684d745e30b3e06473cea1d10",
"published_recovery": "99.0%"
},
{
"model": "Qwen2.5-14B (base)",
"model_id": null,
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor algorithm not stated)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7566,
"quant_score": 0.7544,
"retention": 0.9970922548242136,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-FP8-dynamic/blob/99a1817b207d2d5684d745e30b3e06473cea1d10/README.md#L162",
"source_id": "quant-redhatai-qwen2-5-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-FP8-dynamic/README.md",
"sha256": "a352fd2c712681d5f8b25eb0a28f86f60b187c74bec6912243bb43f1ca2bbade",
"line": 162,
"raw_line": " | 75.44",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-FP8-dynamic/README.md",
"sha256": "a352fd2c712681d5f8b25eb0a28f86f60b187c74bec6912243bb43f1ca2bbade",
"line": 160,
"raw_line": " | 75.66",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose/header mislabel this checkpoint as INT8; config.json at the pinned revision shows FP8 (float, 8-bit) weights and activations, consistent with the repository name. Quant label taken from config.json. Qwen/Qwen2.5-14B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-14B-FP8-dynamic@99a1817:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-FP8-dynamic",
"artifact_revision": "99a1817b207d2d5684d745e30b3e06473cea1d10",
"published_recovery": "99.7%"
},
{
"model": "Qwen2.5-14B-Instruct",
"model_id": "qwen2-5-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7987,
"quant_score": 0.7975,
"retention": 0.9984975585326153,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8/blob/a9d77c2a8d0706cebcc7a5dcdd8221c2bda0ca6d/README.md#L102",
"source_id": "quant-redhatai-qwen2-5-14b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-Instruct-quantized.w8a8/README.md",
"sha256": "ea50ea4902eabcd52d47311cf0bac1e796920a3df285f3b97a1559ede987600a",
"line": 102,
"raw_line": " | 79.75",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-14B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-Instruct-quantized.w8a8/README.md",
"sha256": "ea50ea4902eabcd52d47311cf0bac1e796920a3df285f3b97a1559ede987600a",
"line": 100,
"raw_line": " | 79.87",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-14B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-Instruct-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8@a9d77c2:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8",
"artifact_revision": "a9d77c2a8d0706cebcc7a5dcdd8221c2bda0ca6d",
"published_recovery": "99.9%"
},
{
"model": "Qwen2.5-14B-Instruct",
"model_id": "qwen2-5-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.8355,
"quant_score": 0.8552,
"retention": 1.0235786953919808,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8/blob/a9d77c2a8d0706cebcc7a5dcdd8221c2bda0ca6d/README.md#L122",
"source_id": "quant-redhatai-qwen2-5-14b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-Instruct-quantized.w8a8/README.md",
"sha256": "ea50ea4902eabcd52d47311cf0bac1e796920a3df285f3b97a1559ede987600a",
"line": 122,
"raw_line": " | 85.52",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-14B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-Instruct-quantized.w8a8/README.md",
"sha256": "ea50ea4902eabcd52d47311cf0bac1e796920a3df285f3b97a1559ede987600a",
"line": 120,
"raw_line": " | 83.55",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-14B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-Instruct-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8@a9d77c2:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8",
"artifact_revision": "a9d77c2a8d0706cebcc7a5dcdd8221c2bda0ca6d",
"published_recovery": "102.36%"
},
{
"model": "Qwen2.5-14B-Instruct",
"model_id": "qwen2-5-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7785,
"quant_score": 0.7815,
"retention": 1.0038535645472064,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8/blob/a9d77c2a8d0706cebcc7a5dcdd8221c2bda0ca6d/README.md#L162",
"source_id": "quant-redhatai-qwen2-5-14b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-Instruct-quantized.w8a8/README.md",
"sha256": "ea50ea4902eabcd52d47311cf0bac1e796920a3df285f3b97a1559ede987600a",
"line": 162,
"raw_line": " | 78.15",
"row_label_line": 158,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-14B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-Instruct-quantized.w8a8/README.md",
"sha256": "ea50ea4902eabcd52d47311cf0bac1e796920a3df285f3b97a1559ede987600a",
"line": 160,
"raw_line": " | 77.85",
"row_label_line": 158,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-14B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-Instruct-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8@a9d77c2:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-Instruct-quantized.w8a8",
"artifact_revision": "a9d77c2a8d0706cebcc7a5dcdd8221c2bda0ca6d",
"published_recovery": "100.4%"
},
{
"model": "Qwen2.5-14B (base)",
"model_id": null,
"params_b": 14.8,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7971,
"quant_score": 0.7938,
"retention": 0.9958599924727136,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-quantized.w8a8/blob/b820dc00e4ebd4da94d17d3087a379b2bba40cef/README.md#L102",
"source_id": "quant-redhatai-qwen2-5-14b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-quantized.w8a8/README.md",
"sha256": "1c9b9006df3f13bd97ec5680a5cc2ba6a689f92bd1ea64d1060dcf5c86553fd8",
"line": 102,
"raw_line": " | 79.38",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-14B-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-quantized.w8a8/README.md",
"sha256": "1c9b9006df3f13bd97ec5680a5cc2ba6a689f92bd1ea64d1060dcf5c86553fd8",
"line": 100,
"raw_line": " | 79.71",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-14B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-14B-quantized.w8a8@b820dc0:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-quantized.w8a8",
"artifact_revision": "b820dc00e4ebd4da94d17d3087a379b2bba40cef",
"published_recovery": "99.6%"
},
{
"model": "Qwen2.5-14B (base)",
"model_id": null,
"params_b": 14.8,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.8446,
"quant_score": 0.8393,
"retention": 0.9937248401610231,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-quantized.w8a8/blob/b820dc00e4ebd4da94d17d3087a379b2bba40cef/README.md#L122",
"source_id": "quant-redhatai-qwen2-5-14b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-quantized.w8a8/README.md",
"sha256": "1c9b9006df3f13bd97ec5680a5cc2ba6a689f92bd1ea64d1060dcf5c86553fd8",
"line": 122,
"raw_line": " | 83.93",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-14B-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-quantized.w8a8/README.md",
"sha256": "1c9b9006df3f13bd97ec5680a5cc2ba6a689f92bd1ea64d1060dcf5c86553fd8",
"line": 120,
"raw_line": " | 84.46",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-14B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-14B-quantized.w8a8@b820dc0:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-quantized.w8a8",
"artifact_revision": "b820dc00e4ebd4da94d17d3087a379b2bba40cef",
"published_recovery": "99.4%"
},
{
"model": "Qwen2.5-14B (base)",
"model_id": null,
"params_b": 14.8,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7566,
"quant_score": 0.7543,
"retention": 0.9969600845889508,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-14B-quantized.w8a8/blob/b820dc00e4ebd4da94d17d3087a379b2bba40cef/README.md#L162",
"source_id": "quant-redhatai-qwen2-5-14b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-quantized.w8a8/README.md",
"sha256": "1c9b9006df3f13bd97ec5680a5cc2ba6a689f92bd1ea64d1060dcf5c86553fd8",
"line": 162,
"raw_line": " | 75.43",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-14B-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-14B-quantized.w8a8/README.md",
"sha256": "1c9b9006df3f13bd97ec5680a5cc2ba6a689f92bd1ea64d1060dcf5c86553fd8",
"line": 160,
"raw_line": " | 75.66",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-14B-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-14B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-14B-quantized.w8a8@b820dc0:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-14B-quantized.w8a8",
"artifact_revision": "b820dc00e4ebd4da94d17d3087a379b2bba40cef",
"published_recovery": "99.7%"
},
{
"model": "Qwen2.5-32B (base)",
"model_id": null,
"params_b": 32.8,
"quant": "INT8 W8A16 (weight-only INT8 per-channel; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.8325,
"quant_score": 0.8319,
"retention": 0.9992792792792792,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-32B-quantized.w8a16/blob/718f3680200a636693a0858d145d8651b2264c52/README.md#L100",
"source_id": "quant-redhatai-qwen2-5-32b-w8a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-32B-quantized.w8a16/README.md",
"sha256": "ba5e5da7b88eb8d095e90651c16ea6c60e61aa964cb8bd14085c4c2d689c7b94",
"line": 100,
"raw_line": " | 83.19",
"row_label_line": 96,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-32B-quantized.w8a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-32B-quantized.w8a16/README.md",
"sha256": "ba5e5da7b88eb8d095e90651c16ea6c60e61aa964cb8bd14085c4c2d689c7b94",
"line": 98,
"raw_line": " | 83.25",
"row_label_line": 96,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-32B-quantized.w8a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-32B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-32B-quantized.w8a16@718f368:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-32B-quantized.w8a16",
"artifact_revision": "718f3680200a636693a0858d145d8651b2264c52",
"published_recovery": "99.9%"
},
{
"model": "Qwen2.5-32B (base)",
"model_id": null,
"params_b": 32.8,
"quant": "INT8 W8A16 (weight-only INT8 per-channel; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8k (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7809,
"quant_score": 0.7862,
"retention": 1.0067870405941861,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-32B-quantized.w8a16/blob/718f3680200a636693a0858d145d8651b2264c52/README.md#L120",
"source_id": "quant-redhatai-qwen2-5-32b-w8a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-32B-quantized.w8a16/README.md",
"sha256": "ba5e5da7b88eb8d095e90651c16ea6c60e61aa964cb8bd14085c4c2d689c7b94",
"line": 120,
"raw_line": " | 78.62",
"row_label_line": 116,
"row_label": "GSM-8k (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-32B-quantized.w8a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-32B-quantized.w8a16/README.md",
"sha256": "ba5e5da7b88eb8d095e90651c16ea6c60e61aa964cb8bd14085c4c2d689c7b94",
"line": 118,
"raw_line": " | 78.09",
"row_label_line": 116,
"row_label": "GSM-8k (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-32B-quantized.w8a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-32B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-32B-quantized.w8a16@718f368:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-32B-quantized.w8a16",
"artifact_revision": "718f3680200a636693a0858d145d8651b2264c52",
"published_recovery": "100.7%"
},
{
"model": "Qwen2.5-32B (base)",
"model_id": null,
"params_b": 32.8,
"quant": "INT8 W8A16 (weight-only INT8 per-channel; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8k (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.753,
"quant_score": 0.754,
"retention": 1.00132802124834,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-32B-quantized.w8a16/blob/718f3680200a636693a0858d145d8651b2264c52/README.md#L160",
"source_id": "quant-redhatai-qwen2-5-32b-w8a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-32B-quantized.w8a16/README.md",
"sha256": "ba5e5da7b88eb8d095e90651c16ea6c60e61aa964cb8bd14085c4c2d689c7b94",
"line": 160,
"raw_line": " | 75.40",
"row_label_line": 156,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-32B-quantized.w8a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-32B-quantized.w8a16/README.md",
"sha256": "ba5e5da7b88eb8d095e90651c16ea6c60e61aa964cb8bd14085c4c2d689c7b94",
"line": 158,
"raw_line": " | 75.30",
"row_label_line": 156,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-32B-quantized.w8a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-32B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8k (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-32B-quantized.w8a16@718f368:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-32B-quantized.w8a16",
"artifact_revision": "718f3680200a636693a0858d145d8651b2264c52",
"published_recovery": "100.1%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor algorithm not stated)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7415,
"quant_score": 0.7426,
"retention": 1.0014834794335805,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-FP8-dynamic/blob/972b5206a1ad2891762dbe19e89651f9a7c536bd/README.md#L102",
"source_id": "quant-redhatai-qwen2-5-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-FP8-dynamic/README.md",
"sha256": "0bb5ff2b87796eb4be099d24aadb06a69cb274382ed391ed377a84226579abeb",
"line": 102,
"raw_line": " | 74.26",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-FP8-dynamic/README.md",
"sha256": "0bb5ff2b87796eb4be099d24aadb06a69cb274382ed391ed377a84226579abeb",
"line": 100,
"raw_line": " | 74.15",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-FP8-dynamic@972b520:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-FP8-dynamic",
"artifact_revision": "972b5206a1ad2891762dbe19e89651f9a7c536bd",
"published_recovery": "100.2%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor algorithm not stated)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7976,
"quant_score": 0.8074,
"retention": 1.012286860581745,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-FP8-dynamic/blob/972b5206a1ad2891762dbe19e89651f9a7c536bd/README.md#L122",
"source_id": "quant-redhatai-qwen2-5-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-FP8-dynamic/README.md",
"sha256": "0bb5ff2b87796eb4be099d24aadb06a69cb274382ed391ed377a84226579abeb",
"line": 122,
"raw_line": " | 80.74",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-FP8-dynamic/README.md",
"sha256": "0bb5ff2b87796eb4be099d24aadb06a69cb274382ed391ed377a84226579abeb",
"line": 120,
"raw_line": " | 79.76",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-FP8-dynamic@972b520:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-FP8-dynamic",
"artifact_revision": "972b5206a1ad2891762dbe19e89651f9a7c536bd",
"published_recovery": "101.2%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor algorithm not stated)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7092,
"quant_score": 0.71,
"retention": 1.0011280315848843,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-FP8-dynamic/blob/972b5206a1ad2891762dbe19e89651f9a7c536bd/README.md#L162",
"source_id": "quant-redhatai-qwen2-5-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-FP8-dynamic/README.md",
"sha256": "0bb5ff2b87796eb4be099d24aadb06a69cb274382ed391ed377a84226579abeb",
"line": 162,
"raw_line": " | 71.00",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-FP8-dynamic/README.md",
"sha256": "0bb5ff2b87796eb4be099d24aadb06a69cb274382ed391ed377a84226579abeb",
"line": 160,
"raw_line": " | 70.92",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-7B-FP8-dynamic@972b520:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-FP8-dynamic",
"artifact_revision": "972b5206a1ad2891762dbe19e89651f9a7c536bd",
"published_recovery": "100.1%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7424,
"quant_score": 0.7404,
"retention": 0.9973060344827588,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic/blob/a4f1d5442ea284bc8e5a3b3e1a4d528f7d6caacb/README.md#L355",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-FP8-dynamic/README.md",
"sha256": "ebd2e2b0c5357172f35aafa6570f7222975681d00984575b22f9f4a937a072fb",
"line": 355,
"raw_line": " | 74.04",
"row_label_line": 351,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-Instruct-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-FP8-dynamic/README.md",
"sha256": "ebd2e2b0c5357172f35aafa6570f7222975681d00984575b22f9f4a937a072fb",
"line": 353,
"raw_line": " | 74.24",
"row_label_line": 351,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic@a4f1d54:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic",
"artifact_revision": "a4f1d5442ea284bc8e5a3b3e1a4d528f7d6caacb",
"published_recovery": "99.7%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.8036,
"quant_score": 0.8006,
"retention": 0.9962667994026879,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic/blob/a4f1d5442ea284bc8e5a3b3e1a4d528f7d6caacb/README.md#L375",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-FP8-dynamic/README.md",
"sha256": "ebd2e2b0c5357172f35aafa6570f7222975681d00984575b22f9f4a937a072fb",
"line": 375,
"raw_line": " | 80.06",
"row_label_line": 371,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-Instruct-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-FP8-dynamic/README.md",
"sha256": "ebd2e2b0c5357172f35aafa6570f7222975681d00984575b22f9f4a937a072fb",
"line": 373,
"raw_line": " | 80.36",
"row_label_line": 371,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic@a4f1d54:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic",
"artifact_revision": "a4f1d5442ea284bc8e5a3b3e1a4d528f7d6caacb",
"published_recovery": "99.6%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7316,
"quant_score": 0.7294,
"retention": 0.9969928922908694,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic/blob/a4f1d5442ea284bc8e5a3b3e1a4d528f7d6caacb/README.md#L415",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-FP8-dynamic/README.md",
"sha256": "ebd2e2b0c5357172f35aafa6570f7222975681d00984575b22f9f4a937a072fb",
"line": 415,
"raw_line": " | 72.94",
"row_label_line": 411,
"row_label": "Average",
"column_header": "Qwen2.5-7B-Instruct-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-FP8-dynamic/README.md",
"sha256": "ebd2e2b0c5357172f35aafa6570f7222975681d00984575b22f9f4a937a072fb",
"line": 413,
"raw_line": " | 73.16",
"row_label_line": 411,
"row_label": "Average",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic@a4f1d54:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-FP8-dynamic",
"artifact_revision": "a4f1d5442ea284bc8e5a3b3e1a4d528f7d6caacb",
"published_recovery": "99.7%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7424,
"quant_score": 0.7319,
"retention": 0.9858566810344829,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16/blob/92e73453c88d10975a06b42789f63933a6c3f449/README.md#L369",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w4a16/README.md",
"sha256": "3da1c99b75165a84818ea56fe697a48a7335af5af11a2c4731c9635c5f684574",
"line": 369,
"raw_line": " | 73.19",
"row_label_line": 365,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-Instruct-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w4a16/README.md",
"sha256": "3da1c99b75165a84818ea56fe697a48a7335af5af11a2c4731c9635c5f684574",
"line": 367,
"raw_line": " | 74.24",
"row_label_line": 365,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16@92e7345:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16",
"artifact_revision": "92e73453c88d10975a06b42789f63933a6c3f449",
"published_recovery": "98.6%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.8036,
"quant_score": 0.8059,
"retention": 1.0028621204579393,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16/blob/92e73453c88d10975a06b42789f63933a6c3f449/README.md#L389",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w4a16/README.md",
"sha256": "3da1c99b75165a84818ea56fe697a48a7335af5af11a2c4731c9635c5f684574",
"line": 389,
"raw_line": " | 80.59",
"row_label_line": 385,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-Instruct-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w4a16/README.md",
"sha256": "3da1c99b75165a84818ea56fe697a48a7335af5af11a2c4731c9635c5f684574",
"line": 387,
"raw_line": " | 80.36",
"row_label_line": 385,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16@92e7345:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16",
"artifact_revision": "92e73453c88d10975a06b42789f63933a6c3f449",
"published_recovery": "100.3%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7316,
"quant_score": 0.7269,
"retention": 0.9935757244395845,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16/blob/92e73453c88d10975a06b42789f63933a6c3f449/README.md#L429",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w4a16/README.md",
"sha256": "3da1c99b75165a84818ea56fe697a48a7335af5af11a2c4731c9635c5f684574",
"line": 429,
"raw_line": " | 72.69",
"row_label_line": 425,
"row_label": "Average",
"column_header": "Qwen2.5-7B-Instruct-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w4a16/README.md",
"sha256": "3da1c99b75165a84818ea56fe697a48a7335af5af11a2c4731c9635c5f684574",
"line": 427,
"raw_line": " | 73.16",
"row_label_line": 425,
"row_label": "Average",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16@92e7345:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w4a16",
"artifact_revision": "92e73453c88d10975a06b42789f63933a6c3f449",
"published_recovery": "98.6%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7424,
"quant_score": 0.7387,
"retention": 0.9950161637931035,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8/blob/dae5e0d2d3cc641da65943fa6ef02e5e02fd4d64/README.md#L382",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w8a8/README.md",
"sha256": "7db43356742c41040e3fdbf7323f31027d67e302da3b78a87ea86af7a5732747",
"line": 382,
"raw_line": " | 73.87",
"row_label_line": 378,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w8a8/README.md",
"sha256": "7db43356742c41040e3fdbf7323f31027d67e302da3b78a87ea86af7a5732747",
"line": 380,
"raw_line": " | 74.24",
"row_label_line": 378,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8@dae5e0d:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8",
"artifact_revision": "dae5e0d2d3cc641da65943fa6ef02e5e02fd4d64",
"published_recovery": "99.5%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.8036,
"quant_score": 0.8074,
"retention": 1.0047287207565951,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8/blob/dae5e0d2d3cc641da65943fa6ef02e5e02fd4d64/README.md#L402",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w8a8/README.md",
"sha256": "7db43356742c41040e3fdbf7323f31027d67e302da3b78a87ea86af7a5732747",
"line": 402,
"raw_line": " | 80.74",
"row_label_line": 398,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w8a8/README.md",
"sha256": "7db43356742c41040e3fdbf7323f31027d67e302da3b78a87ea86af7a5732747",
"line": 400,
"raw_line": " | 80.36",
"row_label_line": 398,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8@dae5e0d:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8",
"artifact_revision": "dae5e0d2d3cc641da65943fa6ef02e5e02fd4d64",
"published_recovery": "100.5%"
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7316,
"quant_score": 0.7305,
"retention": 0.9984964461454346,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B-Instruct, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8/blob/dae5e0d2d3cc641da65943fa6ef02e5e02fd4d64/README.md#L442",
"source_id": "quant-redhatai-qwen2-5-7b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w8a8/README.md",
"sha256": "7db43356742c41040e3fdbf7323f31027d67e302da3b78a87ea86af7a5732747",
"line": 442,
"raw_line": " | 73.05",
"row_label_line": 438,
"row_label": "Average",
"column_header": "Qwen2.5-7B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-Instruct-quantized.w8a8/README.md",
"sha256": "7db43356742c41040e3fdbf7323f31027d67e302da3b78a87ea86af7a5732747",
"line": 440,
"raw_line": " | 73.16",
"row_label_line": 438,
"row_label": "Average",
"column_header": "Qwen2.5-7B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-Instruct-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8@dae5e0d:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-Instruct-quantized.w8a8",
"artifact_revision": "dae5e0d2d3cc641da65943fa6ef02e5e02fd4d64",
"published_recovery": "99.4%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7415,
"quant_score": 0.7347,
"retention": 0.9908293998651382,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w4a16/blob/88338db35a8b86592d201e91aac04e5d756500ab/README.md#L157",
"source_id": "quant-redhatai-qwen2-5-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w4a16/README.md",
"sha256": "174229b2830b1d4cbe68e542bd812cfec33403e6668d8d0efd03f2f03bb4349c",
"line": 157,
"raw_line": " | 73.47",
"row_label_line": 153,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w4a16/README.md",
"sha256": "174229b2830b1d4cbe68e542bd812cfec33403e6668d8d0efd03f2f03bb4349c",
"line": 155,
"raw_line": " | 74.15",
"row_label_line": 153,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-quantized.w4a16@88338db:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w4a16",
"artifact_revision": "88338db35a8b86592d201e91aac04e5d756500ab",
"published_recovery": "99.1%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7976,
"quant_score": 0.7908,
"retention": 0.9914744232698093,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w4a16/blob/88338db35a8b86592d201e91aac04e5d756500ab/README.md#L177",
"source_id": "quant-redhatai-qwen2-5-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w4a16/README.md",
"sha256": "174229b2830b1d4cbe68e542bd812cfec33403e6668d8d0efd03f2f03bb4349c",
"line": 177,
"raw_line": " | 79.08",
"row_label_line": 173,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w4a16/README.md",
"sha256": "174229b2830b1d4cbe68e542bd812cfec33403e6668d8d0efd03f2f03bb4349c",
"line": 175,
"raw_line": " | 79.76",
"row_label_line": 173,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-quantized.w4a16@88338db:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w4a16",
"artifact_revision": "88338db35a8b86592d201e91aac04e5d756500ab",
"published_recovery": "99.1%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7092,
"quant_score": 0.7035,
"retention": 0.9919627749576986,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w4a16/blob/88338db35a8b86592d201e91aac04e5d756500ab/README.md#L217",
"source_id": "quant-redhatai-qwen2-5-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w4a16/README.md",
"sha256": "174229b2830b1d4cbe68e542bd812cfec33403e6668d8d0efd03f2f03bb4349c",
"line": 217,
"raw_line": " | 70.35",
"row_label_line": 213,
"row_label": "Average",
"column_header": "Qwen2.5-7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w4a16/README.md",
"sha256": "174229b2830b1d4cbe68e542bd812cfec33403e6668d8d0efd03f2f03bb4349c",
"line": 215,
"raw_line": " | 70.92",
"row_label_line": 213,
"row_label": "Average",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-7B-quantized.w4a16@88338db:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w4a16",
"artifact_revision": "88338db35a8b86592d201e91aac04e5d756500ab",
"published_recovery": "99.2%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT8 W8A16 (weight-only INT8 per-channel; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7415,
"quant_score": 0.7441,
"retention": 1.0035064059339176,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w8a16/blob/0553881e16f1700d3d007227b986661497e8e7bc/README.md#L100",
"source_id": "quant-redhatai-qwen2-5-7b-w8a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a16/README.md",
"sha256": "3610f0e877c8524f2aa26638f31206c3d680e8d833512b776dcfc6cb8168fce5",
"line": 100,
"raw_line": " | 74.41",
"row_label_line": 96,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-7B-quantized.w8a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a16/README.md",
"sha256": "3610f0e877c8524f2aa26638f31206c3d680e8d833512b776dcfc6cb8168fce5",
"line": 98,
"raw_line": " | 74.15",
"row_label_line": 96,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w8a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. The card's ARC-Challenge baseline cell reads 50.39 while its Recovery (100.7%) and the Average (70.92) imply 59.39 as in sibling cards; the ARC row is not recorded and the published Average is used as-is. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-quantized.w8a16@0553881:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w8a16",
"artifact_revision": "0553881e16f1700d3d007227b986661497e8e7bc",
"published_recovery": "100.4%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT8 W8A16 (weight-only INT8 per-channel; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8k (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7976,
"quant_score": 0.8044,
"retention": 1.0085255767301904,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w8a16/blob/0553881e16f1700d3d007227b986661497e8e7bc/README.md#L120",
"source_id": "quant-redhatai-qwen2-5-7b-w8a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a16/README.md",
"sha256": "3610f0e877c8524f2aa26638f31206c3d680e8d833512b776dcfc6cb8168fce5",
"line": 120,
"raw_line": " | 80.44",
"row_label_line": 116,
"row_label": "GSM-8k (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-7B-quantized.w8a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a16/README.md",
"sha256": "3610f0e877c8524f2aa26638f31206c3d680e8d833512b776dcfc6cb8168fce5",
"line": 118,
"raw_line": " | 79.76",
"row_label_line": 116,
"row_label": "GSM-8k (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w8a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. The card's ARC-Challenge baseline cell reads 50.39 while its Recovery (100.7%) and the Average (70.92) imply 59.39 as in sibling cards; the ARC row is not recorded and the published Average is used as-is. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-quantized.w8a16@0553881:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w8a16",
"artifact_revision": "0553881e16f1700d3d007227b986661497e8e7bc",
"published_recovery": "100.9%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT8 W8A16 (weight-only INT8 per-channel; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8k (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7092,
"quant_score": 0.711,
"retention": 1.0025380710659897,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w8a16/blob/0553881e16f1700d3d007227b986661497e8e7bc/README.md#L160",
"source_id": "quant-redhatai-qwen2-5-7b-w8a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a16/README.md",
"sha256": "3610f0e877c8524f2aa26638f31206c3d680e8d833512b776dcfc6cb8168fce5",
"line": 160,
"raw_line": " | 71.10",
"row_label_line": 156,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-7B-quantized.w8a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a16/README.md",
"sha256": "3610f0e877c8524f2aa26638f31206c3d680e8d833512b776dcfc6cb8168fce5",
"line": 158,
"raw_line": " | 70.92",
"row_label_line": 156,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w8a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. The card's ARC-Challenge baseline cell reads 50.39 while its Recovery (100.7%) and the Average (70.92) imply 59.39 as in sibling cards; the ARC row is not recorded and the published Average is used as-is. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8k (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-7B-quantized.w8a16@0553881:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w8a16",
"artifact_revision": "0553881e16f1700d3d007227b986661497e8e7bc",
"published_recovery": "100.2%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7415,
"quant_score": 0.7366,
"retention": 0.9933917734322318,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w8a8/blob/5c84ad661e3c4f382aa885cfc32e57c7a78ae915/README.md#L102",
"source_id": "quant-redhatai-qwen2-5-7b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a8/README.md",
"sha256": "251338187a03c7951628ef2fc2fd94ad7d315ec297ba8b70bbad38828651f010",
"line": 102,
"raw_line": " | 73.66",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a8/README.md",
"sha256": "251338187a03c7951628ef2fc2fd94ad7d315ec297ba8b70bbad38828651f010",
"line": 100,
"raw_line": " | 74.15",
"row_label_line": 98,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-quantized.w8a8@5c84ad6:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w8a8",
"artifact_revision": "5c84ad661e3c4f382aa885cfc32e57c7a78ae915",
"published_recovery": "99.4%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7976,
"quant_score": 0.7983,
"retention": 1.000877632898696,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w8a8/blob/5c84ad661e3c4f382aa885cfc32e57c7a78ae915/README.md#L122",
"source_id": "quant-redhatai-qwen2-5-7b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a8/README.md",
"sha256": "251338187a03c7951628ef2fc2fd94ad7d315ec297ba8b70bbad38828651f010",
"line": 122,
"raw_line": " | 79.83",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a8/README.md",
"sha256": "251338187a03c7951628ef2fc2fd94ad7d315ec297ba8b70bbad38828651f010",
"line": 120,
"raw_line": " | 79.76",
"row_label_line": 118,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen2.5-7B-quantized.w8a8@5c84ad6:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w8a8",
"artifact_revision": "5c84ad661e3c4f382aa885cfc32e57c7a78ae915",
"published_recovery": "100.1%"
},
{
"model": "Qwen2.5-7B (base)",
"model_id": null,
"params_b": 7.62,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; algorithm not stated, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 4096, add_bos_token)",
"full": 0.7092,
"quant_score": 0.707,
"retention": 0.996897913141568,
"baseline_quant": "BF16 (unquantized Qwen/Qwen2.5-7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen2.5-7B-quantized.w8a8/blob/5c84ad661e3c4f382aa885cfc32e57c7a78ae915/README.md#L162",
"source_id": "quant-redhatai-qwen2-5-7b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a8/README.md",
"sha256": "251338187a03c7951628ef2fc2fd94ad7d315ec297ba8b70bbad38828651f010",
"line": 162,
"raw_line": " | 70.70",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-7B-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen2.5-7B-quantized.w8a8/README.md",
"sha256": "251338187a03c7951628ef2fc2fd94ad7d315ec297ba8b70bbad38828651f010",
"line": 160,
"raw_line": " | 70.92",
"row_label_line": 158,
"row_label": "Average",
"column_header": "Qwen2.5-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen2.5-7B-quantized.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen2.5-7B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen2.5-7B-quantized.w8a8@5c84ad6:OpenLLM v1",
"artifact": "RedHatAI/Qwen2.5-7B-quantized.w8a8",
"artifact_revision": "5c84ad661e3c4f382aa885cfc32e57c7a78ae915",
"published_recovery": "99.7%"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.4282,
"quant_score": 0.4232,
"retention": 0.9883232134516581,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-0.6B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-0.6B-FP8-dynamic/blob/068a9040b238d65f5c1064f10232bb15b96c0ff0/README.md#L235",
"source_id": "quant-redhatai-qwen3-0-6b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-FP8-dynamic/README.md",
"sha256": "056d53713092b78c70354cf8d1b0bb18de5b79d8bdfad286d81fb4c491b9dfc6",
"line": 235,
"raw_line": " | 42.32",
"row_label_line": 231,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-FP8-dynamic/README.md",
"sha256": "056d53713092b78c70354cf8d1b0bb18de5b79d8bdfad286d81fb4c491b9dfc6",
"line": 233,
"raw_line": " | 42.82",
"row_label_line": 231,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-0.6B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-0.6B-FP8-dynamic@068a904:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-0.6B-FP8-dynamic",
"artifact_revision": "068a9040b238d65f5c1064f10232bb15b96c0ff0",
"published_recovery": "98.8%"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.3778,
"quant_score": 0.3782,
"retention": 1.0010587612493382,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-0.6B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-0.6B-FP8-dynamic/blob/068a9040b238d65f5c1064f10232bb15b96c0ff0/README.md#L295",
"source_id": "quant-redhatai-qwen3-0-6b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-FP8-dynamic/README.md",
"sha256": "056d53713092b78c70354cf8d1b0bb18de5b79d8bdfad286d81fb4c491b9dfc6",
"line": 295,
"raw_line": " | 37.82",
"row_label_line": 291,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-FP8-dynamic/README.md",
"sha256": "056d53713092b78c70354cf8d1b0bb18de5b79d8bdfad286d81fb4c491b9dfc6",
"line": 293,
"raw_line": " | 37.78",
"row_label_line": 291,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-0.6B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-0.6B-FP8-dynamic@068a904:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-0.6B-FP8-dynamic",
"artifact_revision": "068a9040b238d65f5c1064f10232bb15b96c0ff0",
"published_recovery": "100.1%"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.2929,
"quant_score": 0.2576,
"retention": 0.8794810515534313,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-0.6B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-0.6B-FP8-dynamic/blob/068a9040b238d65f5c1064f10232bb15b96c0ff0/README.md#L411",
"source_id": "quant-redhatai-qwen3-0-6b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-FP8-dynamic/README.md",
"sha256": "056d53713092b78c70354cf8d1b0bb18de5b79d8bdfad286d81fb4c491b9dfc6",
"line": 411,
"raw_line": " | 25.76",
"row_label_line": 407,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-0.6B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-FP8-dynamic/README.md",
"sha256": "056d53713092b78c70354cf8d1b0bb18de5b79d8bdfad286d81fb4c491b9dfc6",
"line": 409,
"raw_line": " | 29.29",
"row_label_line": 407,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-0.6B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-0.6B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-0.6B-FP8-dynamic@068a904:Reasoning",
"artifact": "RedHatAI/Qwen3-0.6B-FP8-dynamic",
"artifact_revision": "068a9040b238d65f5c1064f10232bb15b96c0ff0",
"published_recovery": "88.0%"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.4282,
"quant_score": 0.398,
"retention": 0.9294722092480148,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-0.6B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-0.6B-quantized.w4a16/blob/3bceedd23534ac8ef59952f72c94790b27938e42/README.md#L260",
"source_id": "quant-redhatai-qwen3-0-6b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-quantized.w4a16/README.md",
"sha256": "eba094bd289783c41543427a26438ded38f26080c18e536311b30fbad9c88fa7",
"line": 260,
"raw_line": " | 39.80",
"row_label_line": 256,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-quantized.w4a16/README.md",
"sha256": "eba094bd289783c41543427a26438ded38f26080c18e536311b30fbad9c88fa7",
"line": 258,
"raw_line": " | 42.82",
"row_label_line": 256,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-0.6B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-0.6B-quantized.w4a16@3bceedd:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-0.6B-quantized.w4a16",
"artifact_revision": "3bceedd23534ac8ef59952f72c94790b27938e42",
"published_recovery": "93.00%"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.3778,
"quant_score": 0.3619,
"retention": 0.9579142403388036,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-0.6B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-0.6B-quantized.w4a16/blob/3bceedd23534ac8ef59952f72c94790b27938e42/README.md#L320",
"source_id": "quant-redhatai-qwen3-0-6b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-quantized.w4a16/README.md",
"sha256": "eba094bd289783c41543427a26438ded38f26080c18e536311b30fbad9c88fa7",
"line": 320,
"raw_line": " | 36.19",
"row_label_line": 316,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-quantized.w4a16/README.md",
"sha256": "eba094bd289783c41543427a26438ded38f26080c18e536311b30fbad9c88fa7",
"line": 318,
"raw_line": " | 37.78",
"row_label_line": 316,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-0.6B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-0.6B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-0.6B-quantized.w4a16@3bceedd:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-0.6B-quantized.w4a16",
"artifact_revision": "3bceedd23534ac8ef59952f72c94790b27938e42",
"published_recovery": "95.8%"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.2929,
"quant_score": 0.2778,
"retention": 0.9484465687948105,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-0.6B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-0.6B-quantized.w4a16/blob/3bceedd23534ac8ef59952f72c94790b27938e42/README.md#L436",
"source_id": "quant-redhatai-qwen3-0-6b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-quantized.w4a16/README.md",
"sha256": "eba094bd289783c41543427a26438ded38f26080c18e536311b30fbad9c88fa7",
"line": 436,
"raw_line": " | 27.78",
"row_label_line": 432,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-0.6B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-0.6B-quantized.w4a16/README.md",
"sha256": "eba094bd289783c41543427a26438ded38f26080c18e536311b30fbad9c88fa7",
"line": 434,
"raw_line": " | 29.29",
"row_label_line": 432,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-0.6B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-0.6B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-0.6B-quantized.w4a16@3bceedd:Reasoning",
"artifact": "RedHatAI/Qwen3-0.6B-quantized.w4a16",
"artifact_revision": "3bceedd23534ac8ef59952f72c94790b27938e42",
"published_recovery": "94.8%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.5682,
"quant_score": 0.5602,
"retention": 0.9859204505455825,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-FP8-dynamic/blob/e776dcc126bb25d3d5af2b619acaad8a94332a86/README.md#L235",
"source_id": "quant-redhatai-qwen3-1-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 235,
"raw_line": " | 56.02",
"row_label_line": 231,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 233,
"raw_line": " | 56.82",
"row_label_line": 231,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-FP8-dynamic@e776dcc:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-1.7B-FP8-dynamic",
"artifact_revision": "e776dcc126bb25d3d5af2b619acaad8a94332a86",
"published_recovery": "98.6%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.4367,
"quant_score": 0.4147,
"retention": 0.9496221662468514,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-FP8-dynamic/blob/e776dcc126bb25d3d5af2b619acaad8a94332a86/README.md#L255",
"source_id": "quant-redhatai-qwen3-1-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 255,
"raw_line": " | 41.47",
"row_label_line": 251,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 253,
"raw_line": " | 43.67",
"row_label_line": 251,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-FP8-dynamic@e776dcc:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-1.7B-FP8-dynamic",
"artifact_revision": "e776dcc126bb25d3d5af2b619acaad8a94332a86",
"published_recovery": "95.0%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.4982,
"quant_score": 0.4912,
"retention": 0.985949417904456,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-FP8-dynamic/blob/e776dcc126bb25d3d5af2b619acaad8a94332a86/README.md#L295",
"source_id": "quant-redhatai-qwen3-1-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 295,
"raw_line": " | 49.12",
"row_label_line": 291,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 293,
"raw_line": " | 49.82",
"row_label_line": 291,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-1.7B-FP8-dynamic@e776dcc:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-1.7B-FP8-dynamic",
"artifact_revision": "e776dcc126bb25d3d5af2b619acaad8a94332a86",
"published_recovery": "98.6%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.2428,
"quant_score": 0.2327,
"retention": 0.9584019769357495,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-FP8-dynamic/blob/e776dcc126bb25d3d5af2b619acaad8a94332a86/README.md#L367",
"source_id": "quant-redhatai-qwen3-1-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 367,
"raw_line": " | 23.27",
"row_label_line": 363,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-1.7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 365,
"raw_line": " | 24.28",
"row_label_line": 363,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-1.7B-FP8-dynamic@e776dcc:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-1.7B-FP8-dynamic",
"artifact_revision": "e776dcc126bb25d3d5af2b619acaad8a94332a86",
"published_recovery": "95.8%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.4396,
"quant_score": 0.401,
"retention": 0.9121929026387626,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-FP8-dynamic/blob/e776dcc126bb25d3d5af2b619acaad8a94332a86/README.md#L391",
"source_id": "quant-redhatai-qwen3-1-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 391,
"raw_line": " | 40.10",
"row_label_line": 387,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 389,
"raw_line": " | 43.96",
"row_label_line": 387,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-FP8-dynamic@e776dcc:Reasoning",
"artifact": "RedHatAI/Qwen3-1.7B-FP8-dynamic",
"artifact_revision": "e776dcc126bb25d3d5af2b619acaad8a94332a86",
"published_recovery": "91.2%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.3229,
"quant_score": 0.3229,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-FP8-dynamic/blob/e776dcc126bb25d3d5af2b619acaad8a94332a86/README.md#L401",
"source_id": "quant-redhatai-qwen3-1-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 401,
"raw_line": " | 32.29",
"row_label_line": 397,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 399,
"raw_line": " | 32.29",
"row_label_line": 397,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-FP8-dynamic@e776dcc:Reasoning",
"artifact": "RedHatAI/Qwen3-1.7B-FP8-dynamic",
"artifact_revision": "e776dcc126bb25d3d5af2b619acaad8a94332a86",
"published_recovery": "100.0%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.3838,
"quant_score": 0.3889,
"retention": 1.0132881709223553,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-FP8-dynamic/blob/e776dcc126bb25d3d5af2b619acaad8a94332a86/README.md#L411",
"source_id": "quant-redhatai-qwen3-1-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 411,
"raw_line": " | 38.89",
"row_label_line": 407,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-FP8-dynamic/README.md",
"sha256": "0688c1ba0d8902a78bd5110dc3c7cf8e774a16d23b82bf35dd14237a3553da1b",
"line": 409,
"raw_line": " | 38.38",
"row_label_line": 407,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-FP8-dynamic@e776dcc:Reasoning",
"artifact": "RedHatAI/Qwen3-1.7B-FP8-dynamic",
"artifact_revision": "e776dcc126bb25d3d5af2b619acaad8a94332a86",
"published_recovery": "101.3%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.5682,
"quant_score": 0.5513,
"retention": 0.9702569517775431,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L260",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 260,
"raw_line": " | 55.13",
"row_label_line": 256,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 258,
"raw_line": " | 56.82",
"row_label_line": 256,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "97.0%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.4367,
"quant_score": 0.3063,
"retention": 0.7013968399358828,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L280",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 280,
"raw_line": " | 30.63",
"row_label_line": 276,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 278,
"raw_line": " | 43.67",
"row_label_line": 276,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "70.1%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.4982,
"quant_score": 0.4682,
"retention": 0.9397832195905259,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L320",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 320,
"raw_line": " | 46.82",
"row_label_line": 316,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 318,
"raw_line": " | 49.82",
"row_label_line": 316,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "94.0%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.2428,
"quant_score": 0.2236,
"retention": 0.9209225700164744,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L392",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 392,
"raw_line": " | 22.36",
"row_label_line": 388,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 390,
"raw_line": " | 24.28",
"row_label_line": 388,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "92.1%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.4396,
"quant_score": 0.3208,
"retention": 0.72975432211101,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L416",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 416,
"raw_line": " | 32.08",
"row_label_line": 412,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 414,
"raw_line": " | 43.96",
"row_label_line": 412,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:Reasoning",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "73.0%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.3229,
"quant_score": 0.2823,
"retention": 0.8742644781666151,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L426",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 426,
"raw_line": " | 28.23",
"row_label_line": 422,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 424,
"raw_line": " | 32.29",
"row_label_line": 422,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:Reasoning",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "87.4%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.3838,
"quant_score": 0.3485,
"retention": 0.9080250130276185,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L436",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 436,
"raw_line": " | 34.85",
"row_label_line": 432,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 434,
"raw_line": " | 38.38",
"row_label_line": 432,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:Reasoning",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "90.8%"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 64; GPTQ, llm-compressor)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.3344,
"quant_score": 0.264,
"retention": 0.7894736842105264,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-1.7B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-1.7B-quantized.w4a16/blob/f8467681051842a5be10a8478d8cfdc5facb0af3/README.md#L456",
"source_id": "quant-redhatai-qwen3-1-7b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 456,
"raw_line": " | 26.40",
"row_label_line": 452,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-1.7B-quantized.w4a16/README.md",
"sha256": "68084e067fba7b8021b8f6a5a771de91532677808606abb0d03d2ff252bca793",
"line": 454,
"raw_line": " | 33.44",
"row_label_line": 452,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-1.7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-1.7B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-1.7B-quantized.w4a16@f846768:Reasoning",
"artifact": "RedHatAI/Qwen3-1.7B-quantized.w4a16",
"artifact_revision": "f8467681051842a5be10a8478d8cfdc5facb0af3",
"published_recovery": "79.0%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM8K (Strict-Match, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.884,
"quant_score": 0.884,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-block/blob/dc7381d4ba723321bf2db13ac62fc2db50cc7815/README.md#L196",
"source_id": "quant-redhatai-qwen3-14b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 196,
"raw_line": " | 88.40 | ",
"row_label_line": 194,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "nm-testing/Qwen3-14B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 195,
"raw_line": " 88.40 | ",
"row_label_line": 194,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-14B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-block@dc7381d:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-FP8-block",
"artifact_revision": "dc7381d4ba723321bf2db13ac62fc2db50cc7815",
"published_recovery": "100.00"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (Acc, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.7885,
"quant_score": 0.7873,
"retention": 0.9984781230183893,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-block/blob/dc7381d4ba723321bf2db13ac62fc2db50cc7815/README.md#L208",
"source_id": "quant-redhatai-qwen3-14b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 208,
"raw_line": " 78.73 | ",
"row_label_line": 206,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "nm-testing/Qwen3-14B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 207,
"raw_line": " 78.85 | ",
"row_label_line": 206,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-14B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-block@dc7381d:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-FP8-block",
"artifact_revision": "dc7381d4ba723321bf2db13ac62fc2db50cc7815",
"published_recovery": "99.85"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.7479,
"quant_score": 0.7491,
"retention": 1.0016044925792218,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-block/blob/dc7381d4ba723321bf2db13ac62fc2db50cc7815/README.md#L226",
"source_id": "quant-redhatai-qwen3-14b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 226,
"raw_line": " 74.91 | ",
"row_label_line": 224,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "nm-testing/Qwen3-14B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 225,
"raw_line": " 74.79 | ",
"row_label_line": 224,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-14B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "suite average of 6 tasks: ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-14B-FP8-block@dc7381d:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-FP8-block",
"artifact_revision": "dc7381d4ba723321bf2db13ac62fc2db50cc7815",
"published_recovery": "100.16"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3146,
"quant_score": 0.3109,
"retention": 0.9882390336935792,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-block/blob/dc7381d4ba723321bf2db13ac62fc2db50cc7815/README.md#L270",
"source_id": "quant-redhatai-qwen3-14b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 270,
"raw_line": " 31.09 | ",
"row_label_line": 268,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "nm-testing/Qwen3-14B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-block/README.md",
"sha256": "1e82a2200f6ebcedc18581ef5f900b3aa0760f76f7afbddbb2e6a5191351d043",
"line": 269,
"raw_line": " 31.46 | ",
"row_label_line": 268,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "Qwen/Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-14B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "suite average of 6 tasks: IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-14B-FP8-block@dc7381d:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-14B-FP8-block",
"artifact_revision": "dc7381d4ba723321bf2db13ac62fc2db50cc7815",
"published_recovery": "98.82"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7681,
"quant_score": 0.7647,
"retention": 0.9955734930347611,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L235",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 235,
"raw_line": " 76.47",
"row_label_line": 231,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 233,
"raw_line": " | 76.81",
"row_label_line": 231,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "99.6%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6763,
"quant_score": 0.6619,
"retention": 0.9787076741091232,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L255",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 255,
"raw_line": " | 66.19",
"row_label_line": 251,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 253,
"raw_line": " | 67.63",
"row_label_line": 251,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "97.9%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6317,
"quant_score": 0.6338,
"retention": 1.0033243628304576,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L295",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 295,
"raw_line": " | 63.38",
"row_label_line": 291,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 293,
"raw_line": " | 63.17",
"row_label_line": 291,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "100.3%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3878,
"quant_score": 0.3881,
"retention": 1.0007735946364105,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L367",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 367,
"raw_line": " | 38.81",
"row_label_line": 363,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 365,
"raw_line": " | 38.78",
"row_label_line": 363,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "100.1%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7656,
"quant_score": 0.7688,
"retention": 1.0041797283176592,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L391",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 391,
"raw_line": " | 76.88",
"row_label_line": 387,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 389,
"raw_line": " | 76.56",
"row_label_line": 387,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "100.4%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6635,
"quant_score": 0.6698,
"retention": 1.009495101733233,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L401",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 401,
"raw_line": " | 66.98",
"row_label_line": 397,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 399,
"raw_line": " | 66.35",
"row_label_line": 397,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "101.0%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6162,
"quant_score": 0.6414,
"retention": 1.0408958130477117,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L411",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 411,
"raw_line": " | 64.14",
"row_label_line": 407,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 409,
"raw_line": " | 61.62",
"row_label_line": 407,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "104.1%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6084,
"quant_score": 0.6056,
"retention": 0.9953977646285338,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-FP8-dynamic/blob/8bae2a5f4cf23071026e82c6625c0f93dde165f9/README.md#L431",
"source_id": "quant-redhatai-qwen3-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 431,
"raw_line": " | 60.56",
"row_label_line": 427,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-FP8-dynamic/README.md",
"sha256": "302c5d75639d7bcfd2c1ac8df31d558ebbb14513579c64091863e8b4b89439fe",
"line": 429,
"raw_line": " | 60.84",
"row_label_line": 427,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-FP8-dynamic@8bae2a5:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-FP8-dynamic",
"artifact_revision": "8bae2a5f4cf23071026e82c6625c0f93dde165f9",
"published_recovery": "99.5%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'gsm8k' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.887,
"quant_score": 0.8825,
"retention": 0.9949267192784667,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L219",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 219,
"raw_line": " | 88.25 | ",
"row_label_line": 217,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 218,
"raw_line": " 88.70 | ",
"row_label_line": 217,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "99.49"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'mmlu' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7886,
"quant_score": 0.7723,
"retention": 0.9793304590413391,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L231",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 231,
"raw_line": " 77.23 | ",
"row_label_line": 229,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 230,
"raw_line": " 78.86 | ",
"row_label_line": 229,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "97.93"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7447,
"quant_score": 0.7385,
"retention": 0.9916744997985765,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L249",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 249,
"raw_line": " 73.85 | ",
"row_label_line": 247,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 248,
"raw_line": " 74.47 | ",
"row_label_line": 247,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "99.16"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.5278,
"quant_score": 0.5103,
"retention": 0.9668435013262598,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L292",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 292,
"raw_line": " 51.03 | ",
"row_label_line": 290,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 291,
"raw_line": " 52.78 | ",
"row_label_line": 290,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "96.68"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME24 (0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.7586,
"quant_score": 0.6552,
"retention": 0.8636962826258898,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L306",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 306,
"raw_line": " 65.52 | ",
"row_label_line": 304,
"row_label": "AIME24 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 305,
"raw_line": " 75.86 | ",
"row_label_line": 304,
"row_label": "AIME24 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "86.34"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME25 (0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.6897,
"quant_score": 0.6552,
"retention": 0.9499782514136581,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L312",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 312,
"raw_line": " 65.52 | ",
"row_label_line": 310,
"row_label": "AIME25 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 311,
"raw_line": " 68.97 | ",
"row_label_line": 310,
"row_label": "AIME25 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "95.00"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA (Diamond, 0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.6497,
"quant_score": 0.604,
"retention": 0.9296598430044637,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L318",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 318,
"raw_line": " 60.40 | ",
"row_label_line": 316,
"row_label": "GPQA (Diamond, 0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 317,
"raw_line": " 64.97 | ",
"row_label_line": 316,
"row_label": "GPQA (Diamond, 0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "93.00"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "Reasoning average of 3 tasks (AIME24 (0-shot); AIME25 (0-shot); GPQA (Diamond, 0-shot)) as published; lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.6993,
"quant_score": 0.6381,
"retention": 0.9124839124839125,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-NVFP4/blob/cc120e7bc68557313002275d51db65096ca24303/README.md#L324",
"source_id": "quant-redhatai-qwen3-14b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 324,
"raw_line": " 63.81 | ",
"row_label_line": 322,
"row_label": "Average",
"row_category": "Reasoning",
"column_header": "Qwen3-14B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-NVFP4/README.md",
"sha256": "76e615585399f32eff05fe7f2f5d95741ffbdb0d2d7b14f2172cf682bc8bf8ab",
"line": 323,
"raw_line": " 69.93 | ",
"row_label_line": 322,
"row_label": "Average",
"row_category": "Reasoning",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "suite average of 3 tasks: AIME24 (0-shot); AIME25 (0-shot); GPQA (Diamond, 0-shot)",
"study": "RedHatAI/Qwen3-14B-NVFP4@cc120e7:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-NVFP4",
"artifact_revision": "cc120e7bc68557313002275d51db65096ca24303",
"published_recovery": "91.45"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7681,
"quant_score": 0.759,
"retention": 0.9881525842989194,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L247",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 247,
"raw_line": " 75.90",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 245,
"raw_line": " | 76.81",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "98.8%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6763,
"quant_score": 0.7157,
"retention": 1.0582581694514268,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L267",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 267,
"raw_line": " | 71.57",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 265,
"raw_line": " | 67.63",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "105.8%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6317,
"quant_score": 0.6444,
"retention": 1.0201044799746715,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L307",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 307,
"raw_line": " | 64.44",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 305,
"raw_line": " | 63.17",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "102.0%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3878,
"quant_score": 0.3859,
"retention": 0.9951005673027333,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L379",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 379,
"raw_line": " | 38.59",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 377,
"raw_line": " | 38.78",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "99.5%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7656,
"quant_score": 0.7792,
"retention": 1.0177638453500522,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L403",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 403,
"raw_line": " | 77.92",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 401,
"raw_line": " | 76.56",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "101.8%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6635,
"quant_score": 0.65,
"retention": 0.9796533534287868,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L413",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 413,
"raw_line": " | 65.00",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 411,
"raw_line": " | 66.35",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "98.0%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6162,
"quant_score": 0.6111,
"retention": 0.9917234664070107,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L423",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 423,
"raw_line": " | 61.11",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 421,
"raw_line": " | 61.62",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "99.2%"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6084,
"quant_score": 0.5856,
"retention": 0.9625246548323471,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-14B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-14B-quantized.w4a16/blob/98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685/README.md#L443",
"source_id": "quant-redhatai-qwen3-14b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 443,
"raw_line": " | 58.56",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-14B-quantized.w4a16/README.md",
"sha256": "f5c9af70c3065787018f26d9dfd6bea5efdcdd91cd961e0f89df93727d2d457d",
"line": 441,
"raw_line": " | 60.84",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-14B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-14B-quantized.w4a16@98ce4ba:Reasoning",
"artifact": "RedHatAI/Qwen3-14B-quantized.w4a16",
"artifact_revision": "98ce4ba52bc02e37e2beb0b0a5f11d7f0e589685",
"published_recovery": "96.3%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM8K (Strict-Match, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.8999,
"quant_score": 0.8916,
"retention": 0.9907767529725525,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-block/blob/6b2399be90eff484848cbbabd684930e01400d54/README.md#L196",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 196,
"raw_line": " | 89.16 | ",
"row_label_line": 194,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-30B-A3B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 195,
"raw_line": " 89.99 | ",
"row_label_line": 194,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-30B-A3B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-block@6b2399b:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-block",
"artifact_revision": "6b2399be90eff484848cbbabd684930e01400d54",
"published_recovery": "99.07"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (Acc, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.795,
"quant_score": 0.7937,
"retention": 0.9983647798742139,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-block/blob/6b2399be90eff484848cbbabd684930e01400d54/README.md#L208",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 208,
"raw_line": " 79.37 | ",
"row_label_line": 206,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-30B-A3B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 207,
"raw_line": " 79.50 | ",
"row_label_line": 206,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-30B-A3B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-block@6b2399b:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-block",
"artifact_revision": "6b2399be90eff484848cbbabd684930e01400d54",
"published_recovery": "99.84"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.7365,
"quant_score": 0.7374,
"retention": 1.0012219959266802,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-block/blob/6b2399be90eff484848cbbabd684930e01400d54/README.md#L226",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 226,
"raw_line": " 73.74 | ",
"row_label_line": 224,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-30B-A3B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 225,
"raw_line": " 73.65 | ",
"row_label_line": 224,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-30B-A3B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-block@6b2399b:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-block",
"artifact_revision": "6b2399be90eff484848cbbabd684930e01400d54",
"published_recovery": "100.12"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3098,
"quant_score": 0.3162,
"retention": 1.0206584893479664,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-block/blob/6b2399be90eff484848cbbabd684930e01400d54/README.md#L270",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 270,
"raw_line": " 31.62 | ",
"row_label_line": 268,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "RedHatAI/Qwen3-30B-A3B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-block/README.md",
"sha256": "d92fd3b5f3463381c4793ff7c8c5423de6d9f76f5611f43683c1af5b84b32a29",
"line": 269,
"raw_line": " 30.98 | ",
"row_label_line": 268,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "Qwen/Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-30B-A3B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-block@6b2399b:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-block",
"artifact_revision": "6b2399be90eff484848cbbabd684930e01400d54",
"published_recovery": "102.07"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7767,
"quant_score": 0.7749,
"retention": 0.9976825028968712,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L161",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 161,
"raw_line": " 77.49",
"row_label_line": 157,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 159,
"raw_line": " | 77.67",
"row_label_line": 157,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "99.8%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.8726,
"quant_score": 0.8673,
"retention": 0.9939261975704791,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L181",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 181,
"raw_line": " | 86.73",
"row_label_line": 177,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 179,
"raw_line": " | 87.26",
"row_label_line": 177,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "99.4%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6762,
"quant_score": 0.6756,
"retention": 0.9991126885536823,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L221",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 221,
"raw_line": " | 67.56",
"row_label_line": 217,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 219,
"raw_line": " | 67.62",
"row_label_line": 217,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "99.9%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (settings not published in card); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3818,
"quant_score": 0.3975,
"retention": 1.0411210057621794,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L293",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 293,
"raw_line": " | 39.75",
"row_label_line": 289,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 291,
"raw_line": " | 38.18",
"row_label_line": 289,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. This card only documents the OpenLLM v1 command; settings for its v2/reasoning rows are not published in the card. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "104.1%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval; settings not published in card",
"full": 0.7833,
"quant_score": 0.7896,
"retention": 1.0080428954423593,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L317",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 317,
"raw_line": " | 78.96",
"row_label_line": 313,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 315,
"raw_line": " | 78.33",
"row_label_line": 313,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. This card only documents the OpenLLM v1 command; settings for its v2/reasoning rows are not published in the card. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "100.8%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval; settings not published in card",
"full": 0.7146,
"quant_score": 0.6844,
"retention": 0.9577385950181921,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L327",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 327,
"raw_line": " | 68.44",
"row_label_line": 323,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 325,
"raw_line": " | 71.46",
"row_label_line": 323,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. This card only documents the OpenLLM v1 command; settings for its v2/reasoning rows are not published in the card. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "95.8%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval; settings not published in card",
"full": 0.6263,
"quant_score": 0.6263,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L337",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 337,
"raw_line": " | 62.63",
"row_label_line": 333,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 335,
"raw_line": " | 62.63",
"row_label_line": 333,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. This card only documents the OpenLLM v1 command; settings for its v2/reasoning rows are not published in the card. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "100.0%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval; settings not published in card",
"full": 0.6066,
"quant_score": 0.6089,
"retention": 1.0037916254533465,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-FP8-dynamic/blob/10a439027a4fd228adc03a7bffb1aac10a179938/README.md#L357",
"source_id": "quant-redhatai-qwen3-30b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 357,
"raw_line": " | 60.89",
"row_label_line": 353,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-FP8-dynamic/README.md",
"sha256": "6232eaf99a201698da72675fc5a8301ac0a278985bc334e13f9b9974504d4313",
"line": 355,
"raw_line": " | 60.66",
"row_label_line": 353,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. This card only documents the OpenLLM v1 command; settings for its v2/reasoning rows are not published in the card. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic@10a4390:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-FP8-dynamic",
"artifact_revision": "10a439027a4fd228adc03a7bffb1aac10a179938",
"published_recovery": "100.4%"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (5-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9611,
"quant_score": 0.9578,
"retention": 0.9965664342940381,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/blob/e9c59cdcca9d73f24adeda09660af904a79f509f/README.md#L212",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 212,
"raw_line": "| GSM8k Platinum (5-shot) | 96.11 | 95.78 | 99.66 |",
"column_header": "Qwen3-30B-A3B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 212,
"raw_line": "| GSM8k Platinum (5-shot) | 96.11 | 95.78 | 99.66 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16@e9c59cd:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16",
"artifact_revision": "e9c59cdcca9d73f24adeda09660af904a79f509f",
"published_recovery": "99.66"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU (CoT) — published row 'MMLU-Cot (5-shot)'; lm-evaluation-harness task mmlu_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8429,
"quant_score": 0.835,
"retention": 0.9906275952070233,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/blob/e9c59cdcca9d73f24adeda09660af904a79f509f/README.md#L213",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 213,
"raw_line": "| MMLU-Cot (5-shot) | 84.29 | 83.50 | 99.07 |",
"column_header": "Qwen3-30B-A3B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 213,
"raw_line": "| MMLU-Cot (5-shot) | 84.29 | 83.50 | 99.07 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Cot (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16@e9c59cd:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16",
"artifact_revision": "e9c59cdcca9d73f24adeda09660af904a79f509f",
"published_recovery": "99.07"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (5-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.789,
"quant_score": 0.7711,
"retention": 0.9773130544993662,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/blob/e9c59cdcca9d73f24adeda09660af904a79f509f/README.md#L214",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 214,
"raw_line": "| MMLU-Pro (5-shot) | 78.90 | 77.11 | 97.73 |",
"column_header": "Qwen3-30B-A3B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 214,
"raw_line": "| MMLU-Pro (5-shot) | 78.90 | 77.11 | 97.73 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16@e9c59cd:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16",
"artifact_revision": "e9c59cdcca9d73f24adeda09660af904a79f509f",
"published_recovery": "97.73"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8913,
"quant_score": 0.8913,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/blob/e9c59cdcca9d73f24adeda09660af904a79f509f/README.md#L215",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 215,
"raw_line": "| IfEval | 89.13 | 89.13| 100.0 |",
"column_header": "Qwen3-30B-A3B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 215,
"raw_line": "| IfEval | 89.13 | 89.13| 100.0 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16@e9c59cd:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16",
"artifact_revision": "e9c59cdcca9d73f24adeda09660af904a79f509f",
"published_recovery": "100.0"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8991,
"quant_score": 0.922,
"retention": 1.0254699143588033,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/blob/e9c59cdcca9d73f24adeda09660af904a79f509f/README.md#L216",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 216,
"raw_line": "| Math 500 | 89.91 | 92.20 | 102.54 |",
"column_header": "Qwen3-30B-A3B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "abee3c59f0cda10a0cec6d33103debe065e608d376f65a126d4a3933a1042726",
"line": 216,
"raw_line": "| Math 500 | 89.91 | 92.20 | 102.54 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16@e9c59cd:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w4a16",
"artifact_revision": "e9c59cdcca9d73f24adeda09660af904a79f509f",
"published_recovery": "102.54"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (5-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9611,
"quant_score": 0.9757,
"retention": 1.0151909270627406,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/blob/b2b37fb1c4cd5a5769b841ef1f672627b8304d8e/README.md#L225",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 225,
"raw_line": "| GSM8k Platinum (5-shot) | 96.11 | 97.57 | 101.52 |",
"column_header": "Qwen3-30B-A3B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 225,
"raw_line": "| GSM8k Platinum (5-shot) | 96.11 | 97.57 | 101.52 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8@b2b37fb:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8",
"artifact_revision": "b2b37fb1c4cd5a5769b841ef1f672627b8304d8e",
"published_recovery": "101.52"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MMLU (CoT) — published row 'MMLU-Cot (5-shot)'; lm-evaluation-harness task mmlu_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8429,
"quant_score": 0.843,
"retention": 1.000118638035354,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/blob/b2b37fb1c4cd5a5769b841ef1f672627b8304d8e/README.md#L226",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 226,
"raw_line": "| MMLU-Cot (5-shot) | 84.29 | 84.30 | 100.02 |",
"column_header": "Qwen3-30B-A3B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 226,
"raw_line": "| MMLU-Cot (5-shot) | 84.29 | 84.30 | 100.02 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Cot (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8@b2b37fb:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8",
"artifact_revision": "b2b37fb1c4cd5a5769b841ef1f672627b8304d8e",
"published_recovery": "100.02"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (5-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.789,
"quant_score": 0.7881,
"retention": 0.9988593155893536,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/blob/b2b37fb1c4cd5a5769b841ef1f672627b8304d8e/README.md#L227",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 227,
"raw_line": "| MMLU-Pro (5-shot) | 78.90 | 78.81 | 99.89 |",
"column_header": "Qwen3-30B-A3B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 227,
"raw_line": "| MMLU-Pro (5-shot) | 78.90 | 78.81 | 99.89 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8@b2b37fb:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8",
"artifact_revision": "b2b37fb1c4cd5a5769b841ef1f672627b8304d8e",
"published_recovery": "99.89"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8913,
"quant_score": 0.8889,
"retention": 0.997307303938068,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/blob/b2b37fb1c4cd5a5769b841ef1f672627b8304d8e/README.md#L228",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 228,
"raw_line": "| IfEval | 89.13 | 88.89 | 99.73 |",
"column_header": "Qwen3-30B-A3B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 228,
"raw_line": "| IfEval | 89.13 | 88.89 | 99.73 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8@b2b37fb:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8",
"artifact_revision": "b2b37fb1c4cd5a5769b841ef1f672627b8304d8e",
"published_recovery": "99.73"
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8991,
"quant_score": 0.9048,
"retention": 1.0063396730063396,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/blob/b2b37fb1c4cd5a5769b841ef1f672627b8304d8e/README.md#L229",
"source_id": "quant-redhatai-qwen3-30b-a3b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 229,
"raw_line": "| Math 500 | 89.91 | 90.48 | 100.62 |",
"column_header": "Qwen3-30B-A3B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "88b8afdd042c920032ddd3c6281489ac56d197ee8b4d6a28954080cd281e76f4",
"line": 229,
"raw_line": "| Math 500 | 89.91 | 90.48 | 100.62 |",
"column_header": "Qwen3-30B-A3B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8@b2b37fb:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8",
"artifact_revision": "b2b37fb1c4cd5a5769b841ef1f672627b8304d8e",
"published_recovery": "100.62"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "GSM8K — published row 'gsm8k' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.8946,
"quant_score": 0.8772,
"retention": 0.9805499664654594,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-NVFP4/blob/a4df9f017835830cbb588cfe72bc4e50294183c0/README.md#L217",
"source_id": "quant-redhatai-qwen3-30b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 217,
"raw_line": " | 87.72 | ",
"row_label_line": 215,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-30B-A3B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 216,
"raw_line": " 89.46 | ",
"row_label_line": 215,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-NVFP4@a4df9f0:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-NVFP4",
"artifact_revision": "a4df9f017835830cbb588cfe72bc4e50294183c0",
"published_recovery": "98.05"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "MMLU — published row 'mmlu' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7951,
"quant_score": 0.7754,
"retention": 0.9752232423594517,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-NVFP4/blob/a4df9f017835830cbb588cfe72bc4e50294183c0/README.md#L229",
"source_id": "quant-redhatai-qwen3-30b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 229,
"raw_line": " 77.54 | ",
"row_label_line": 227,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-30B-A3B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 228,
"raw_line": " 79.51 | ",
"row_label_line": 227,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-NVFP4@a4df9f0:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-NVFP4",
"artifact_revision": "a4df9f017835830cbb588cfe72bc4e50294183c0",
"published_recovery": "97.52"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7325,
"quant_score": 0.7192,
"retention": 0.9818430034129693,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-NVFP4/blob/a4df9f017835830cbb588cfe72bc4e50294183c0/README.md#L247",
"source_id": "quant-redhatai-qwen3-30b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 247,
"raw_line": " 71.92 | ",
"row_label_line": 245,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-30B-A3B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 246,
"raw_line": " 73.25 | ",
"row_label_line": 245,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande",
"study": "RedHatAI/Qwen3-30B-A3B-NVFP4@a4df9f0:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-NVFP4",
"artifact_revision": "a4df9f017835830cbb588cfe72bc4e50294183c0",
"published_recovery": "98.19"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.5278,
"quant_score": 0.4899,
"retention": 0.9281924971580143,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-NVFP4/blob/a4df9f017835830cbb588cfe72bc4e50294183c0/README.md#L290",
"source_id": "quant-redhatai-qwen3-30b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 290,
"raw_line": " 48.99 | ",
"row_label_line": 288,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen3-30B-A3B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-NVFP4/README.md",
"sha256": "a3e375ae4027274c4b6686d98e6d556710067a0080d1de44425fcafaad09479f",
"line": 289,
"raw_line": " 52.78 | ",
"row_label_line": 288,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)",
"study": "RedHatAI/Qwen3-30B-A3B-NVFP4@a4df9f0:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-30B-A3B-NVFP4",
"artifact_revision": "a4df9f017835830cbb588cfe72bc4e50294183c0",
"published_recovery": "92.81"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (0-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8555,
"quant_score": 0.7265,
"retention": 0.8492109877264759,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/blob/98ffbaed9192e305240ff9fb6647ac1ea0abb467/README.md#L206",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 206,
"raw_line": "| GSM8k Platinum (0-shot) | 85.55 | 72.65 | 84.91 |",
"column_header": "Qwen3-30B-A3B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 206,
"raw_line": "| GSM8k Platinum (0-shot) | 85.55 | 72.65 | 84.91 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16@98ffbae:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16",
"artifact_revision": "98ffbaed9192e305240ff9fb6647ac1ea0abb467",
"published_recovery": "84.91"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (0-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8071,
"quant_score": 0.7906,
"retention": 0.9795564366249536,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/blob/98ffbaed9192e305240ff9fb6647ac1ea0abb467/README.md#L207",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 207,
"raw_line": "| MMLU-Pro (0-shot) | 80.71 | 79.06 | 97.96 |",
"column_header": "Qwen3-30B-A3B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 207,
"raw_line": "| MMLU-Pro (0-shot) | 80.71 | 79.06 | 97.96 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16@98ffbae:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16",
"artifact_revision": "98ffbaed9192e305240ff9fb6647ac1ea0abb467",
"published_recovery": "97.96"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval (0-shot)'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9009,
"quant_score": 0.9029,
"retention": 1.0022200022200023,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/blob/98ffbaed9192e305240ff9fb6647ac1ea0abb467/README.md#L208",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 208,
"raw_line": "| IfEval (0-shot) | 90.09 | 90.29 | 100.22 |",
"column_header": "Qwen3-30B-A3B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 208,
"raw_line": "| IfEval (0-shot) | 90.09 | 90.29 | 100.22 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16@98ffbae:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16",
"artifact_revision": "98ffbaed9192e305240ff9fb6647ac1ea0abb467",
"published_recovery": "100.22"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8667,
"quant_score": 0.8333,
"retention": 0.9614630206530517,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/blob/98ffbaed9192e305240ff9fb6647ac1ea0abb467/README.md#L209",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 209,
"raw_line": "| AIME 2025 | 86.67 | 83.33 | 96.15 |",
"column_header": "Qwen3-30B-A3B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 209,
"raw_line": "| AIME 2025 | 86.67 | 83.33 | 96.15 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16@98ffbae:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16",
"artifact_revision": "98ffbaed9192e305240ff9fb6647ac1ea0abb467",
"published_recovery": "96.15"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.715,
"quant_score": 0.7159,
"retention": 1.0012587412587413,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/blob/98ffbaed9192e305240ff9fb6647ac1ea0abb467/README.md#L210",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 210,
"raw_line": "| GPQA diamond | 71.50 | 71.59 | 100.12 |",
"column_header": "Qwen3-30B-A3B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 210,
"raw_line": "| GPQA diamond | 71.50 | 71.59 | 100.12 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16@98ffbae:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16",
"artifact_revision": "98ffbaed9192e305240ff9fb6647ac1ea0abb467",
"published_recovery": "100.12"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.9075,
"quant_score": 0.931,
"retention": 1.0258953168044076,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/blob/98ffbaed9192e305240ff9fb6647ac1ea0abb467/README.md#L211",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 211,
"raw_line": "| Math 500 | 90.75 | 93.10 | 102.59 |",
"column_header": "Qwen3-30B-A3B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "03894549ef5ecf9ded7d2adc0fce28b0f64b5504fbfd66f7f427d536f19cdaf3",
"line": 211,
"raw_line": "| Math 500 | 90.75 | 93.10 | 102.59 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16@98ffbae:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w4a16",
"artifact_revision": "98ffbaed9192e305240ff9fb6647ac1ea0abb467",
"published_recovery": "102.59"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (0-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8555,
"quant_score": 0.8381,
"retention": 0.9796610169491528,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/blob/fb485fe24f99380b6f480ac767f671cd5c740908/README.md#L216",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 216,
"raw_line": "| GSM8k Platinum (0-shot) | 85.55 | 83.81 | 97.97 |",
"column_header": "Qwen3-30B-A3B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 216,
"raw_line": "| GSM8k Platinum (0-shot) | 85.55 | 83.81 | 97.97 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8@fb485fe:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8",
"artifact_revision": "fb485fe24f99380b6f480ac767f671cd5c740908",
"published_recovery": "97.97"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (0-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8071,
"quant_score": 0.8066,
"retention": 0.9993804980795441,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/blob/fb485fe24f99380b6f480ac767f671cd5c740908/README.md#L217",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 217,
"raw_line": "| MMLU-Pro (0-shot) | 80.71 | 80.66 | 99.94 |",
"column_header": "Qwen3-30B-A3B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 217,
"raw_line": "| MMLU-Pro (0-shot) | 80.71 | 80.66 | 99.94 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8@fb485fe:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8",
"artifact_revision": "fb485fe24f99380b6f480ac767f671cd5c740908",
"published_recovery": "99.94"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval (0-shot)'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9009,
"quant_score": 0.9013,
"retention": 1.0004440004440005,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/blob/fb485fe24f99380b6f480ac767f671cd5c740908/README.md#L218",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 218,
"raw_line": "| IfEval (0-shot) | 90.09 | 90.13 | 100.04 |",
"column_header": "Qwen3-30B-A3B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 218,
"raw_line": "| IfEval (0-shot) | 90.09 | 90.13 | 100.04 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8@fb485fe:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8",
"artifact_revision": "fb485fe24f99380b6f480ac767f671cd5c740908",
"published_recovery": "100.04"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8667,
"quant_score": 0.8417,
"retention": 0.9711549555786315,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/blob/fb485fe24f99380b6f480ac767f671cd5c740908/README.md#L219",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 219,
"raw_line": "| AIME 2025 | 86.67 | 84.17 | 97.11 |",
"column_header": "Qwen3-30B-A3B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 219,
"raw_line": "| AIME 2025 | 86.67 | 84.17 | 97.11 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8@fb485fe:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8",
"artifact_revision": "fb485fe24f99380b6f480ac767f671cd5c740908",
"published_recovery": "97.11"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.715,
"quant_score": 0.7241,
"retention": 1.0127272727272727,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/blob/fb485fe24f99380b6f480ac767f671cd5c740908/README.md#L220",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 220,
"raw_line": "| GPQA diamond | 71.50 | 72.41 | 101.27 |",
"column_header": "Qwen3-30B-A3B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 220,
"raw_line": "| GPQA diamond | 71.50 | 72.41 | 101.27 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8@fb485fe:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8",
"artifact_revision": "fb485fe24f99380b6f480ac767f671cd5c740908",
"published_recovery": "101.27"
},
{
"model": "Qwen3-30B-A3B-Thinking-2507",
"model_id": "qwen3-30b-a3b-thinking-2507",
"params_b": 30.5,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.9075,
"quant_score": 0.9063,
"retention": 0.9986776859504133,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/blob/fb485fe24f99380b6f480ac767f671cd5c740908/README.md#L221",
"source_id": "quant-redhatai-qwen3-30b-a3b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 221,
"raw_line": "| Math 500 | 90.75 | 90.63 | 99.86 |",
"column_header": "Qwen3-30B-A3B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "2c29ecca5ed98942a8d895a562f1bdee2093c74d7635a34c386ae4a7e61ea9e5",
"line": 221,
"raw_line": "| Math 500 | 90.75 | 90.63 | 99.86 |",
"column_header": "Qwen3-30B-A3B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8@fb485fe:card table",
"artifact": "RedHatAI/Qwen3-30B-A3B-Thinking-2507-quantized.w8a8",
"artifact_revision": "fb485fe24f99380b6f480ac767f671cd5c740908",
"published_recovery": "99.86"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7767,
"quant_score": 0.7611,
"retention": 0.9799150251062185,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L247",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 247,
"raw_line": " 76.11",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 245,
"raw_line": " | 77.67",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "98.00%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.8726,
"quant_score": 0.8666,
"retention": 0.9931239972495988,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L267",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 267,
"raw_line": " | 86.66",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 265,
"raw_line": " | 87.26",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "99.3%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6762,
"quant_score": 0.666,
"retention": 0.9849157054125997,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L307",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 307,
"raw_line": " | 66.60",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 305,
"raw_line": " | 67.62",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "98.5%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3818,
"quant_score": 0.375,
"retention": 0.9821896280775275,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L379",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 379,
"raw_line": " | 37.50",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 377,
"raw_line": " | 38.18",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "98.2%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7833,
"quant_score": 0.7854,
"retention": 1.0026809651474533,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L403",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 403,
"raw_line": " | 78.54",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 401,
"raw_line": " | 78.33",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "100.3%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7146,
"quant_score": 0.7031,
"retention": 0.983907080884411,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L413",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 413,
"raw_line": " | 70.31",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 411,
"raw_line": " | 71.46",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "98.4%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6263,
"quant_score": 0.6212,
"retention": 0.9918569375698545,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L423",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 423,
"raw_line": " | 62.12",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 421,
"raw_line": " | 62.63",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "99.2%"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6066,
"quant_score": 0.5875,
"retention": 0.968513023409166,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-30B-A3B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-30B-A3B-quantized.w4a16/blob/1c842a5000febac4b335c94a04789a9d4756a18e/README.md#L443",
"source_id": "quant-redhatai-qwen3-30b-a3b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 443,
"raw_line": " | 58.75",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-30B-A3B-quantized.w4a16/README.md",
"sha256": "942d0da5e3865016baaaba1787db00a48c5f05ff343aa1cbf1622f12007252c0",
"line": 441,
"raw_line": " | 60.66",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-30B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-30B-A3B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-30B-A3B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16@1c842a5:Reasoning",
"artifact": "RedHatAI/Qwen3-30B-A3B-quantized.w4a16",
"artifact_revision": "1c842a5000febac4b335c94a04789a9d4756a18e",
"published_recovery": "96.9%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM8K (Strict-Match, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.7415,
"quant_score": 0.7453,
"retention": 1.0051247471341873,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-block/blob/90c5558f4bc66d558f691d0b3077d39b1dd70dec/README.md#L197",
"source_id": "quant-redhatai-qwen3-32b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 197,
"raw_line": " | 74.53 | ",
"row_label_line": 195,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "nm-testing/Qwen3-32B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 196,
"raw_line": " 74.15 | ",
"row_label_line": 195,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-32B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-block@90c5558:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-FP8-block",
"artifact_revision": "90c5558f4bc66d558f691d0b3077d39b1dd70dec",
"published_recovery": "100.51"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (Acc, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.8199,
"quant_score": 0.8197,
"retention": 0.999756067813148,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-block/blob/90c5558f4bc66d558f691d0b3077d39b1dd70dec/README.md#L209",
"source_id": "quant-redhatai-qwen3-32b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 209,
"raw_line": " 81.97 | ",
"row_label_line": 207,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "nm-testing/Qwen3-32B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 208,
"raw_line": " 81.99 | ",
"row_label_line": 207,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-32B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-block@90c5558:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-FP8-block",
"artifact_revision": "90c5558f4bc66d558f691d0b3077d39b1dd70dec",
"published_recovery": "99.97"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.7472,
"quant_score": 0.7451,
"retention": 0.9971895074946469,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-block/blob/90c5558f4bc66d558f691d0b3077d39b1dd70dec/README.md#L227",
"source_id": "quant-redhatai-qwen3-32b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 227,
"raw_line": " 74.51 | ",
"row_label_line": 225,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "nm-testing/Qwen3-32B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 226,
"raw_line": " 74.72 | ",
"row_label_line": 225,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-32B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "suite average of 6 tasks: ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-32B-FP8-block@90c5558:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-FP8-block",
"artifact_revision": "90c5558f4bc66d558f691d0b3077d39b1dd70dec",
"published_recovery": "99.72"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.305,
"quant_score": 0.2962,
"retention": 0.9711475409836067,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-block/blob/90c5558f4bc66d558f691d0b3077d39b1dd70dec/README.md#L271",
"source_id": "quant-redhatai-qwen3-32b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 271,
"raw_line": " 29.62 | ",
"row_label_line": 269,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "nm-testing/Qwen3-32B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-block/README.md",
"sha256": "5a151f6c16b2f8f8f812e7dfab328321016dbe14d26ec89d0755bdacb3a6f522",
"line": 270,
"raw_line": " 30.50 | ",
"row_label_line": 269,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"nm-testing/Qwen3-32B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "suite average of 6 tasks: IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-32B-FP8-block@90c5558:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-32B-FP8-block",
"artifact_revision": "90c5558f4bc66d558f691d0b3077d39b1dd70dec",
"published_recovery": "97.11"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.8096,
"quant_score": 0.8089,
"retention": 0.999135375494071,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L236",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 236,
"raw_line": " 80.89",
"row_label_line": 232,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 234,
"raw_line": " | 80.96",
"row_label_line": 232,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "99.9%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.8764,
"quant_score": 0.8832,
"retention": 1.0077590141487904,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L256",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 256,
"raw_line": " | 88.32",
"row_label_line": 252,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 254,
"raw_line": " | 87.64",
"row_label_line": 252,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "100.8%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7286,
"quant_score": 0.7294,
"retention": 1.0010979961570132,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L296",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 296,
"raw_line": " | 72.94",
"row_label_line": 292,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 294,
"raw_line": " | 72.86",
"row_label_line": 292,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "100.1%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.4213,
"quant_score": 0.4271,
"retention": 1.0137669119392358,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L368",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 368,
"raw_line": " | 42.71",
"row_label_line": 364,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 366,
"raw_line": " | 42.13",
"row_label_line": 364,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "101.4%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7937,
"quant_score": 0.7937,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L392",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 392,
"raw_line": " | 79.37",
"row_label_line": 388,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 390,
"raw_line": " | 79.37",
"row_label_line": 388,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "100.0%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7177,
"quant_score": 0.7042,
"retention": 0.9811899122195904,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L402",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 402,
"raw_line": " | 70.42",
"row_label_line": 398,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 400,
"raw_line": " | 71.77",
"row_label_line": 398,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "98.1%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6667,
"quant_score": 0.6869,
"retention": 1.030298485075746,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L412",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 412,
"raw_line": " | 68.69",
"row_label_line": 408,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 410,
"raw_line": " | 66.67",
"row_label_line": 408,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "103.0%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6245,
"quant_score": 0.6332,
"retention": 1.0139311449159327,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-FP8-dynamic/blob/c6732fc26128341172e4005bad34aafa51c32866/README.md#L432",
"source_id": "quant-redhatai-qwen3-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 432,
"raw_line": " | 63.32",
"row_label_line": 428,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-FP8-dynamic/README.md",
"sha256": "c95479147168f1817d8e76d995d951268a3cf355bb36598270b071b4cb2cd3e0",
"line": 430,
"raw_line": " | 62.45",
"row_label_line": 428,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-FP8-dynamic@c6732fc:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-FP8-dynamic",
"artifact_revision": "c6732fc26128341172e4005bad34aafa51c32866",
"published_recovery": "101.4%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'gsm8k' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7415,
"quant_score": 0.7468,
"retention": 1.0071476736345246,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L193",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 193,
"raw_line": " | 74.68 | ",
"row_label_line": 191,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 192,
"raw_line": " 74.15 | ",
"row_label_line": 191,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "100.71"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'mmlu' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.8184,
"quant_score": 0.8123,
"retention": 0.9925464320625611,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L205",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 205,
"raw_line": " 81.23 | ",
"row_label_line": 203,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 204,
"raw_line": " 81.84 | ",
"row_label_line": 203,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "99.25"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7432,
"quant_score": 0.742,
"retention": 0.9983853606027987,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L223",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 223,
"raw_line": " 74.20 | ",
"row_label_line": 221,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 222,
"raw_line": " 74.32 | ",
"row_label_line": 221,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "99.83"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.5569,
"quant_score": 0.5445,
"retention": 0.9777338840007184,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L266",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 266,
"raw_line": " 54.45 | ",
"row_label_line": 264,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 265,
"raw_line": " 55.69 | ",
"row_label_line": 264,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "97.79"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME24 (0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.7586,
"quant_score": 0.6897,
"retention": 0.9091747956762457,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L280",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 280,
"raw_line": " 68.97 | ",
"row_label_line": 278,
"row_label": "AIME24 (0-shot)",
"row_category": "Reasoning",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 279,
"raw_line": " 75.86 | ",
"row_label_line": 278,
"row_label": "AIME24 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "90.93"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME25 (0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.7241,
"quant_score": 0.6552,
"retention": 0.9048473967684022,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L286",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 286,
"raw_line": " 65.52 | ",
"row_label_line": 284,
"row_label": "AIME25 (0-shot)",
"row_category": "Reasoning",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 285,
"raw_line": " 72.41 | ",
"row_label_line": 284,
"row_label": "AIME25 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "90.52"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA (Diamond, 0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.6294,
"quant_score": 0.6447,
"retention": 1.0243088655862727,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L292",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 292,
"raw_line": " 64.47 | ",
"row_label_line": 290,
"row_label": "GPQA (Diamond, 0-shot)",
"row_category": "Reasoning",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 291,
"raw_line": " 62.94 | ",
"row_label_line": 290,
"row_label": "GPQA (Diamond, 0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "102.43"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor QuantizationModifier)",
"metric": "Reasoning average of 3 tasks (AIME24 (0-shot); AIME25 (0-shot); GPQA (Diamond, 0-shot)) as published; lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.704,
"quant_score": 0.6632,
"retention": 0.9420454545454543,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4/blob/10a4cab9378ab938be865492b96ff42faff11c3f/README.md#L298",
"source_id": "quant-redhatai-qwen3-32b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 298,
"raw_line": " 66.32 | ",
"row_label_line": 296,
"row_label": "Average",
"row_category": "Reasoning",
"column_header": "RedHatAI/Qwen3-32B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4/README.md",
"sha256": "31a62c410ceca553affa8d80ea6d6d0ddf68def85e57387c272bd24ea436ea50",
"line": 297,
"raw_line": " 70.40 | ",
"row_label_line": 296,
"row_label": "Average",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "suite average of 3 tasks: AIME24 (0-shot); AIME25 (0-shot); GPQA (Diamond, 0-shot)",
"study": "RedHatAI/Qwen3-32B-NVFP4@10a4cab:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-NVFP4",
"artifact_revision": "10a4cab9378ab938be865492b96ff42faff11c3f",
"published_recovery": "94.21"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4A16 (FP4 weights, group 16, 16-bit activations; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.8094,
"quant_score": 0.8057,
"retention": 0.995428712626637,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4A16/blob/c572e992f6042d2b6bfa6853f55b8196a4c8cf44/README.md#L180",
"source_id": "quant-redhatai-qwen3-32b-nvfp4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 180,
"raw_line": " 80.57 | ",
"row_label_line": 178,
"row_label": "MMLU",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-32B-NVFP4A16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 179,
"raw_line": " 80.94 | ",
"row_label_line": 178,
"row_label": "MMLU",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4A16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-NVFP4A16@c572e99:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-NVFP4A16",
"artifact_revision": "c572e992f6042d2b6bfa6853f55b8196a4c8cf44",
"published_recovery": "99.55%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4A16 (FP4 weights, group 16, 16-bit activations; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM8K (8-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.8734,
"quant_score": 0.8772,
"retention": 1.0043508129150445,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4A16/blob/c572e992f6042d2b6bfa6853f55b8196a4c8cf44/README.md#L192",
"source_id": "quant-redhatai-qwen3-32b-nvfp4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 192,
"raw_line": " 87.72 | ",
"row_label_line": 190,
"row_label": "GSM8K (8-shot, strict-match)",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-32B-NVFP4A16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 191,
"raw_line": " 87.34 | ",
"row_label_line": 190,
"row_label": "GSM8K (8-shot, strict-match)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4A16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-NVFP4A16@c572e99:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-NVFP4A16",
"artifact_revision": "c572e992f6042d2b6bfa6853f55b8196a4c8cf44",
"published_recovery": "100.43%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4A16 (FP4 weights, group 16, 16-bit activations; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU; ARC Challenge (0-shot); GSM8K (8-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7272,
"quant_score": 0.7271,
"retention": 0.9998624862486248,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4A16/blob/c572e992f6042d2b6bfa6853f55b8196a4c8cf44/README.md#L216",
"source_id": "quant-redhatai-qwen3-32b-nvfp4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 216,
"raw_line": " 72.71 | ",
"row_label_line": 214,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-32B-NVFP4A16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 215,
"raw_line": " 72.72 | ",
"row_label_line": 214,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4A16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU; ARC Challenge (0-shot); GSM8K (8-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-32B-NVFP4A16@c572e99:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-NVFP4A16",
"artifact_revision": "c572e992f6042d2b6bfa6853f55b8196a4c8cf44",
"published_recovery": "99.98%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "NVFP4A16 (FP4 weights, group 16, 16-bit activations; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-|v|-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.554,
"quant_score": 0.5488,
"retention": 0.9906137184115525,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-NVFP4A16/blob/c572e992f6042d2b6bfa6853f55b8196a4c8cf44/README.md#L259",
"source_id": "quant-redhatai-qwen3-32b-nvfp4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 259,
"raw_line": " 54.88 | ",
"row_label_line": 257,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "RedHatAI/Qwen3-32B-NVFP4A16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-NVFP4A16/README.md",
"sha256": "53909a49069119b65db2d80b532f79c806ebdf4de94f71af1d03dc4c0699451d",
"line": 258,
"raw_line": " 55.40 | ",
"row_label_line": 257,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen/Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-32B-NVFP4A16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-|v|-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-32B-NVFP4A16@c572e99:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-32B-NVFP4A16",
"artifact_revision": "c572e992f6042d2b6bfa6853f55b8196a4c8cf44",
"published_recovery": "99.06%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.8096,
"quant_score": 0.8036,
"retention": 0.9925889328063241,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L247",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 247,
"raw_line": " 80.36",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 245,
"raw_line": " | 80.96",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "99.3%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.8764,
"quant_score": 0.8597,
"retention": 0.9809447740757645,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L267",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 267,
"raw_line": " | 85.97",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 265,
"raw_line": " | 87.64",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "98.1%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7286,
"quant_score": 0.7252,
"retention": 0.9953335163326927,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L307",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 307,
"raw_line": " | 72.52",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 305,
"raw_line": " | 72.86",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "99.6%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.4213,
"quant_score": 0.4124,
"retention": 0.9788749109897936,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L379",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 379,
"raw_line": " | 41.24",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 377,
"raw_line": " | 42.13",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "97.9%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7937,
"quant_score": 0.7729,
"retention": 0.9737936247952627,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L403",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 403,
"raw_line": " | 77.29",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 401,
"raw_line": " | 79.37",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "97.4%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7177,
"quant_score": 0.6427,
"retention": 0.8954995123310574,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L413",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 413,
"raw_line": " | 64.27",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 411,
"raw_line": " | 71.77",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "89.6%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6667,
"quant_score": 0.6667,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L423",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 423,
"raw_line": " | 66.67",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 421,
"raw_line": " | 66.67",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "100.0%"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6245,
"quant_score": 0.5963,
"retention": 0.9548438751000801,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-32B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-32B-quantized.w4a16/blob/0364e6546d36447655a78bac8ab181572fd6ff24/README.md#L443",
"source_id": "quant-redhatai-qwen3-32b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 443,
"raw_line": " | 59.63",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-32B-quantized.w4a16/README.md",
"sha256": "7249e06983fa8637d26e5fafce276cbcfa71ef7b3a8d30d7db3c04b1b5630cf9",
"line": 441,
"raw_line": " | 62.45",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-32B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-32B-quantized.w4a16@0364e65:Reasoning",
"artifact": "RedHatAI/Qwen3-32B-quantized.w4a16",
"artifact_revision": "0364e6546d36447655a78bac8ab181572fd6ff24",
"published_recovery": "95.5%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6676,
"quant_score": 0.6634,
"retention": 0.9937088076692628,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L231",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 231,
"raw_line": " | 66.34",
"row_label_line": 227,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 229,
"raw_line": " | 66.76",
"row_label_line": 227,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "99.4%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.608,
"quant_score": 0.6611,
"retention": 1.0873355263157896,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L251",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 251,
"raw_line": " | 66.11",
"row_label_line": 247,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 249,
"raw_line": " | 60.80",
"row_label_line": 247,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "108.7%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.5679,
"quant_score": 0.5799,
"retention": 1.0211304807184363,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L291",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 291,
"raw_line": " | 57.99",
"row_label_line": 287,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 289,
"raw_line": " | 56.79",
"row_label_line": 287,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "102.1%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3393,
"quant_score": 0.3349,
"retention": 0.9870321249631596,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L363",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 363,
"raw_line": " | 33.49",
"row_label_line": 359,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 361,
"raw_line": " | 33.93",
"row_label_line": 359,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "98.7%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7135,
"quant_score": 0.6937,
"retention": 0.9722494744218643,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L387",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 387,
"raw_line": " | 69.37",
"row_label_line": 383,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 385,
"raw_line": " | 71.35",
"row_label_line": 383,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "97.2%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5958,
"quant_score": 0.6073,
"retention": 1.0193017791205101,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L397",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 397,
"raw_line": " | 60.73",
"row_label_line": 393,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 395,
"raw_line": " | 59.58",
"row_label_line": 393,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "101.9%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5556,
"quant_score": 0.5404,
"retention": 0.97264218862491,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L407",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 407,
"raw_line": " | 54.04",
"row_label_line": 403,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 405,
"raw_line": " | 55.56",
"row_label_line": 403,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "97.3%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5303,
"quant_score": 0.5145,
"retention": 0.9702055440316804,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-FP8-dynamic/blob/31ad70c60365f34890b3118d38d1ad4dce9a6d99/README.md#L427",
"source_id": "quant-redhatai-qwen3-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 427,
"raw_line": " | 51.45",
"row_label_line": 423,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-FP8-dynamic/README.md",
"sha256": "eaeab9e9b3d423cce6635ac5ec2b0cac0c5556225854933b8901e70d606cfc32",
"line": 425,
"raw_line": " | 53.03",
"row_label_line": 423,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-FP8-dynamic@31ad70c:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-FP8-dynamic",
"artifact_revision": "31ad70c60365f34890b3118d38d1ad4dce9a6d99",
"published_recovery": "97.0%"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (5-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9562,
"quant_score": 0.9614,
"retention": 1.0054381928466847,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16/blob/5d4fde88f26ef2577a3e660db6304eaa6cbe99a1/README.md#L205",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 205,
"raw_line": "| GSM8k Platinum (5-shot) | 95.62 | 96.14 | 100.55 |",
"column_header": "Qwen3-4B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 205,
"raw_line": "| GSM8k Platinum (5-shot) | 95.62 | 96.14 | 100.55 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16@5d4fde8:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16",
"artifact_revision": "5d4fde88f26ef2577a3e660db6304eaa6cbe99a1",
"published_recovery": "100.55"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU (CoT) — published row 'MMLU-CoT (5-shot)'; lm-evaluation-harness task mmlu_cot_llama via vLLM server (command not found in card)",
"full": 0.7755,
"quant_score": 0.7683,
"retention": 0.990715667311412,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16/blob/5d4fde88f26ef2577a3e660db6304eaa6cbe99a1/README.md#L206",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 206,
"raw_line": "| MMLU-CoT (5-shot) | 77.55 | 76.83 | 99.08 |",
"column_header": "Qwen3-4B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 206,
"raw_line": "| MMLU-CoT (5-shot) | 77.55 | 76.83 | 99.08 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-CoT (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16@5d4fde8:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16",
"artifact_revision": "5d4fde88f26ef2577a3e660db6304eaa6cbe99a1",
"published_recovery": "99.08"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (5-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.7013,
"quant_score": 0.6894,
"retention": 0.9830315129046059,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16/blob/5d4fde88f26ef2577a3e660db6304eaa6cbe99a1/README.md#L207",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 207,
"raw_line": "| MMLU-Pro (5-shot) | 70.13 | 68.94 | 98.31 |",
"column_header": "Qwen3-4B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 207,
"raw_line": "| MMLU-Pro (5-shot) | 70.13 | 68.94 | 98.31 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16@5d4fde8:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16",
"artifact_revision": "5d4fde88f26ef2577a3e660db6304eaa6cbe99a1",
"published_recovery": "98.31"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8901,
"quant_score": 0.8861,
"retention": 0.9955061229075385,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16/blob/5d4fde88f26ef2577a3e660db6304eaa6cbe99a1/README.md#L208",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 208,
"raw_line": "| IfEval | 89.01 | 88.61 | 99.55 |",
"column_header": "Qwen3-4B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 208,
"raw_line": "| IfEval | 89.01 | 88.61 | 99.55 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16@5d4fde8:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16",
"artifact_revision": "5d4fde88f26ef2577a3e660db6304eaa6cbe99a1",
"published_recovery": "99.55"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8433,
"quant_score": 0.842,
"retention": 0.9984584370923754,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16/blob/5d4fde88f26ef2577a3e660db6304eaa6cbe99a1/README.md#L209",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 209,
"raw_line": "| Math 500 | 84.33 | 84.20 | 99.85 |",
"column_header": "Qwen3-4B Instruct.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w4a16/README.md",
"sha256": "da3a84211782b24659f4b05f24a09ad07bc215c8e240959bafeec9bd3f7047e0",
"line": 209,
"raw_line": "| Math 500 | 84.33 | 84.20 | 99.85 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16@5d4fde8:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w4a16",
"artifact_revision": "5d4fde88f26ef2577a3e660db6304eaa6cbe99a1",
"published_recovery": "99.85"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (5-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9562,
"quant_score": 0.9573,
"retention": 1.0011503869483371,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8/blob/31ec5013b3319d7ba82b2a354d6e48b376204dd3/README.md#L224",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 224,
"raw_line": "| GSM8k Platinum (5-shot) | 95.62 | 95.73 | 100.12 |",
"column_header": "Qwen3-4B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 224,
"raw_line": "| GSM8k Platinum (5-shot) | 95.62 | 95.73 | 100.12 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8@31ec501:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8",
"artifact_revision": "31ec5013b3319d7ba82b2a354d6e48b376204dd3",
"published_recovery": "100.12"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MMLU (CoT) — published row 'MMLU-CoT (5-shot)'; lm-evaluation-harness task mmlu_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.7755,
"quant_score": 0.7704,
"retention": 0.993423597678917,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8/blob/31ec5013b3319d7ba82b2a354d6e48b376204dd3/README.md#L225",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 225,
"raw_line": "| MMLU-CoT (5-shot) | 77.55 | 77.04 | 99.34 |",
"column_header": "Qwen3-4B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 225,
"raw_line": "| MMLU-CoT (5-shot) | 77.55 | 77.04 | 99.34 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-CoT (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8@31ec501:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8",
"artifact_revision": "31ec5013b3319d7ba82b2a354d6e48b376204dd3",
"published_recovery": "99.34"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (5-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.7013,
"quant_score": 0.6996,
"retention": 0.9975759304149436,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8/blob/31ec5013b3319d7ba82b2a354d6e48b376204dd3/README.md#L226",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 226,
"raw_line": "| MMLU-Pro (5-shot) | 70.13 | 69.96 | 99.75 |",
"column_header": "Qwen3-4B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 226,
"raw_line": "| MMLU-Pro (5-shot) | 70.13 | 69.96 | 99.75 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (5-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8@31ec501:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8",
"artifact_revision": "31ec5013b3319d7ba82b2a354d6e48b376204dd3",
"published_recovery": "99.75"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.8901,
"quant_score": 0.8905,
"retention": 1.0004493877092462,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8/blob/31ec5013b3319d7ba82b2a354d6e48b376204dd3/README.md#L227",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 227,
"raw_line": "| IfEval |89.01 | 89.05| 100.04 |",
"column_header": "Qwen3-4B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 227,
"raw_line": "| IfEval |89.01 | 89.05| 100.04 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8@31ec501:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8",
"artifact_revision": "31ec5013b3319d7ba82b2a354d6e48b376204dd3",
"published_recovery": "100.04"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.4762,
"quant_score": 0.4667,
"retention": 0.9800503989920203,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8/blob/31ec5013b3319d7ba82b2a354d6e48b376204dd3/README.md#L228",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 228,
"raw_line": "| AIME 2025 | 47.62 | 46.67 | 98.00 |",
"column_header": "Qwen3-4B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 228,
"raw_line": "| AIME 2025 | 47.62 | 46.67 | 98.00 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8@31ec501:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8",
"artifact_revision": "31ec5013b3319d7ba82b2a354d6e48b376204dd3",
"published_recovery": "98.00"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.452,
"quant_score": 0.452,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8/blob/31ec5013b3319d7ba82b2a354d6e48b376204dd3/README.md#L229",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 229,
"raw_line": "| GPQA diamond | 45.20 | 45.20 | 100.00 |",
"column_header": "Qwen3-4B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 229,
"raw_line": "| GPQA diamond | 45.20 | 45.20 | 100.00 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8@31ec501:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8",
"artifact_revision": "31ec5013b3319d7ba82b2a354d6e48b376204dd3",
"published_recovery": "100.00"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8433,
"quant_score": 0.8363,
"retention": 0.991699276651251,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Instruct-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8/blob/31ec5013b3319d7ba82b2a354d6e48b376204dd3/README.md#L230",
"source_id": "quant-redhatai-qwen3-4b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 230,
"raw_line": "| Math 500 | 84.33 | 83.63| 99.17 |",
"column_header": "Qwen3-4B Instruct.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "5706cb4b1dfb56d672f81fb5b972ceeab77297991f740bb1c1c7aef8ddd82362",
"line": 230,
"raw_line": "| Math 500 | 84.33 | 83.63| 99.17 |",
"column_header": "Qwen3-4B Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Instruct.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8@31ec501:card table",
"artifact": "RedHatAI/Qwen3-4B-Instruct-2507-quantized.w8a8",
"artifact_revision": "31ec5013b3319d7ba82b2a354d6e48b376204dd3",
"published_recovery": "99.17"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (0-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9526,
"quant_score": 0.9487,
"retention": 0.9959059416334244,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16/blob/6131664544d9d715c78f3b6c092fbe5568e2e79e/README.md#L205",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 205,
"raw_line": "| GSM8k Platinum (0-shot) | 95.26 | 94.87 | 99.59 |",
"column_header": "Qwen3-4B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 205,
"raw_line": "| GSM8k Platinum (0-shot) | 95.26 | 94.87 | 99.59 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16@6131664:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16",
"artifact_revision": "6131664544d9d715c78f3b6c092fbe5568e2e79e",
"published_recovery": "99.59"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (0-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.7363,
"quant_score": 0.7133,
"retention": 0.968762732581828,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16/blob/6131664544d9d715c78f3b6c092fbe5568e2e79e/README.md#L206",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 206,
"raw_line": "| MMLU-Pro (0-shot) | 73.63 | 71.33 | 96.87 |",
"column_header": "Qwen3-4B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 206,
"raw_line": "| MMLU-Pro (0-shot) | 73.63 | 71.33 | 96.87 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16@6131664:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16",
"artifact_revision": "6131664544d9d715c78f3b6c092fbe5568e2e79e",
"published_recovery": "96.87"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval (0-shot)'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9017,
"quant_score": 0.9021,
"retention": 1.0004436065210156,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16/blob/6131664544d9d715c78f3b6c092fbe5568e2e79e/README.md#L207",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 207,
"raw_line": "| IfEval (0-shot) |90.17 | 90.21 | 100.04 |",
"column_header": "Qwen3-4B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 207,
"raw_line": "| IfEval (0-shot) |90.17 | 90.21 | 100.04 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16@6131664:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16",
"artifact_revision": "6131664544d9d715c78f3b6c092fbe5568e2e79e",
"published_recovery": "100.04"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.7375,
"quant_score": 0.7208,
"retention": 0.9773559322033898,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16/blob/6131664544d9d715c78f3b6c092fbe5568e2e79e/README.md#L208",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 208,
"raw_line": "| AIME 2025 | 73.75 | 72.08 | 97.74 |",
"column_header": "Qwen3-4B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 208,
"raw_line": "| AIME 2025 | 73.75 | 72.08 | 97.74 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16@6131664:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16",
"artifact_revision": "6131664544d9d715c78f3b6c092fbe5568e2e79e",
"published_recovery": "97.74"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.6894,
"quant_score": 0.6326,
"retention": 0.9176095155207425,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16/blob/6131664544d9d715c78f3b6c092fbe5568e2e79e/README.md#L209",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 209,
"raw_line": "| GPQA diamond | 68.94 | 63.26 | 91.76 |",
"column_header": "Qwen3-4B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 209,
"raw_line": "| GPQA diamond | 68.94 | 63.26 | 91.76 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16@6131664:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16",
"artifact_revision": "6131664544d9d715c78f3b6c092fbe5568e2e79e",
"published_recovery": "91.76"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8945,
"quant_score": 0.9063,
"retention": 1.0131917272219115,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16/blob/6131664544d9d715c78f3b6c092fbe5568e2e79e/README.md#L210",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 210,
"raw_line": "| Math 500 | 89.45 | 90.63 | 101.31 |",
"column_header": "Qwen3-4B Thinking.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w4a16/README.md",
"sha256": "c0a44086c9c67badea64c7b25c18a7e84f563c09392d5413899101154f3052f7",
"line": 210,
"raw_line": "| Math 500 | 89.45 | 90.63 | 101.31 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16@6131664:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w4a16",
"artifact_revision": "6131664544d9d715c78f3b6c092fbe5568e2e79e",
"published_recovery": "101.31"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GSM8K-Platinum — published row 'GSM8k Platinum (0-shot)'; lm-evaluation-harness task gsm8k_platinum_cot_llama via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9526,
"quant_score": 0.9578,
"retention": 1.0054587444887675,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8/blob/a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c/README.md#L215",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 215,
"raw_line": "| GSM8k Platinum (0-shot) | 95.26 | 95.78 | 100.55 |",
"column_header": "Qwen3-4B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 215,
"raw_line": "| GSM8k Platinum (0-shot) | 95.26 | 95.78 | 100.55 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"GSM8k Platinum (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8@a6d3fbb:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8",
"artifact_revision": "a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c",
"published_recovery": "100.55"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MMLU-Pro — published row 'MMLU-Pro (0-shot)'; lm-evaluation-harness task mmlu_pro_chat via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.7363,
"quant_score": 0.7271,
"retention": 0.9875050930327313,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8/blob/a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c/README.md#L216",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 216,
"raw_line": "| MMLU-Pro (0-shot) | 73.63 | 72.71 | 98.74 |",
"column_header": "Qwen3-4B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 216,
"raw_line": "| MMLU-Pro (0-shot) | 73.63 | 72.71 | 98.74 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Table label says \"MMLU-Pro (0-shot)\" but the published lm_eval commands use --num_fewshot 0 with chat completions; shot count ambiguous.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8@a6d3fbb:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8",
"artifact_revision": "a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c",
"published_recovery": "98.74"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "IFEval — published row 'IfEval (0-shot)'; lm-evaluation-harness task ifeval via vLLM OpenAI-compatible server (backend local-chat-completions, chat template, --num_fewshot 0, gen_kwargs do_sample=True,temperature=0.6,top_p=0.95,top_k=20,min_p=0.0,max_gen_toks=64000)",
"full": 0.9017,
"quant_score": 0.8981,
"retention": 0.9960075413108572,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8/blob/a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c/README.md#L217",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 217,
"raw_line": "| IfEval (0-shot) |90.17 | 89.81| 99.60 |",
"column_header": "Qwen3-4B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 217,
"raw_line": "| IfEval (0-shot) |90.17 | 89.81| 99.60 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8@a6d3fbb:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8",
"artifact_revision": "a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c",
"published_recovery": "99.60"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.7375,
"quant_score": 0.7167,
"retention": 0.9717966101694915,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8/blob/a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c/README.md#L218",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 218,
"raw_line": "| AIME 2025 | 73.75 | 71.67 | 97.18 |",
"column_header": "Qwen3-4B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 218,
"raw_line": "| AIME 2025 | 73.75 | 71.67 | 97.18 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8@a6d3fbb:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8",
"artifact_revision": "a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c",
"published_recovery": "97.18"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.6894,
"quant_score": 0.6667,
"retention": 0.9670728169422688,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8/blob/a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c/README.md#L219",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 219,
"raw_line": "| GPQA diamond | 68.94 | 66.67 | 96.70 |",
"column_header": "Qwen3-4B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 219,
"raw_line": "| GPQA diamond | 68.94 | 66.67 | 96.70 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8@a6d3fbb:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8",
"artifact_revision": "a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c",
"published_recovery": "96.70"
},
{
"model": "Qwen3-4B-Thinking-2507",
"model_id": "qwen3-4b-thinking-2507",
"params_b": 4.02,
"quant": "INT8 W8A8 (weights INT8 per-channel, activations INT8 per-token dynamic; SmoothQuant+GPTQ, llm-compressor)",
"metric": "MATH-500 — published row 'Math 500'; lighteval (litellm endpoint to vLLM server), temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32000",
"full": 0.8945,
"quant_score": 0.8928,
"retention": 0.9980994969256568,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B-Thinking-2507, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8/blob/a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c/README.md#L220",
"source_id": "quant-redhatai-qwen3-4b-thinking-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 220,
"raw_line": "| Math 500 | 89.45 | 89.28| 99.80 |",
"column_header": "Qwen3-4B Thinking.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-Thinking-2507-quantized.w8a8/README.md",
"sha256": "bbe6490e2328ba66740bc257a04d97e73353b7748d992074022d57eabbca32bd",
"line": 220,
"raw_line": "| Math 500 | 89.45 | 89.28| 99.80 |",
"column_header": "Qwen3-4B Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B Thinking.w8a8 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8@a6d3fbb:card table",
"artifact": "RedHatAI/Qwen3-4B-Thinking-2507-quantized.w8a8",
"artifact_revision": "a6d3fbb1c3cd5c34ce1bdd4e244e04647f48691c",
"published_recovery": "99.80"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6676,
"quant_score": 0.6501,
"retention": 0.9737866986219292,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L247",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 247,
"raw_line": " | 65.01",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 245,
"raw_line": " | 66.76",
"row_label_line": 243,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "97.38%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.608,
"quant_score": 0.6118,
"retention": 1.00625,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L267",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 267,
"raw_line": " | 61.18",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 265,
"raw_line": " | 60.80",
"row_label_line": 263,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "100.6%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.5679,
"quant_score": 0.5763,
"retention": 1.0147913365029055,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L307",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 307,
"raw_line": " | 57.63",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 305,
"raw_line": " | 56.79",
"row_label_line": 303,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "101.5%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3393,
"quant_score": 0.3257,
"retention": 0.9599174771588564,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L379",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 379,
"raw_line": " | 32.57",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 377,
"raw_line": " | 33.93",
"row_label_line": 375,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "96.0%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7135,
"quant_score": 0.6385,
"retention": 0.8948843728100913,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L403",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 403,
"raw_line": " | 63.85",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 401,
"raw_line": " | 71.35",
"row_label_line": 399,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "89.5%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5998,
"quant_score": 0.5771,
"retention": 0.9621540513504503,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L413",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 413,
"raw_line": " | 57.71",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 411,
"raw_line": " | 59.98",
"row_label_line": 409,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "96.9%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5556,
"quant_score": 0.5354,
"retention": 0.9636429085673146,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L423",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 423,
"raw_line": " | 53.54",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 421,
"raw_line": " | 55.56",
"row_label_line": 419,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "96.4%"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "INT4 W4A16 (weight-only INT4, symmetric, group 128; GPTQ, llm-compressor)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5303,
"quant_score": 0.4951,
"retention": 0.9336224778427306,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-4B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-4B-quantized.w4a16/blob/7a37568f5fcaf2ad3f84cdb9a1bb612546645614/README.md#L443",
"source_id": "quant-redhatai-qwen3-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 443,
"raw_line": " | 49.51",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-4B-quantized.w4a16/README.md",
"sha256": "e7c07970c1d922a0fb7b26e0e7b95d82820ad4bf96e7692a76ff20ff1373ed60",
"line": 441,
"raw_line": " | 53.03",
"row_label_line": 439,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-4B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-4B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=True, group_size=128; quant label follows config.json (weights were uploaded once, before the README). Qwen/Qwen3-4B is not a catalogue model id (base/pre-2507 checkpoint); model_id left null.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-4B-quantized.w4a16@7a37568:Reasoning",
"artifact": "RedHatAI/Qwen3-4B-quantized.w4a16",
"artifact_revision": "7a37568f5fcaf2ad3f84cdb9a1bb612546645614",
"published_recovery": "93.4%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM8K (Strict-Match, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.8795,
"quant_score": 0.8779,
"retention": 0.9981807845366685,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-block/blob/b4fbea251a7905936ad2656faf78d4ca2839d9ad/README.md#L195",
"source_id": "quant-redhatai-qwen3-8b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 195,
"raw_line": " | 87.79 | ",
"row_label_line": 193,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-8B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 194,
"raw_line": " 87.95 | ",
"row_label_line": 193,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-8B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-block@b4fbea2:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-FP8-block",
"artifact_revision": "b4fbea251a7905936ad2656faf78d4ca2839d9ad",
"published_recovery": "99.83"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (Acc, 5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.7488,
"quant_score": 0.747,
"retention": 0.997596153846154,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-block/blob/b4fbea251a7905936ad2656faf78d4ca2839d9ad/README.md#L207",
"source_id": "quant-redhatai-qwen3-8b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 207,
"raw_line": " 74.70 | ",
"row_label_line": 205,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-8B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 206,
"raw_line": " 74.88 | ",
"row_label_line": 205,
"row_label": "MMLU (Acc, 5-shot)",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-8B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-block@b4fbea2:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-FP8-block",
"artifact_revision": "b4fbea251a7905936ad2656faf78d4ca2839d9ad",
"published_recovery": "99.75"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, add_bos_token)",
"full": 0.7212,
"quant_score": 0.7212,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-block/blob/b4fbea251a7905936ad2656faf78d4ca2839d9ad/README.md#L225",
"source_id": "quant-redhatai-qwen3-8b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 225,
"raw_line": " 72.12 | ",
"row_label_line": 223,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "RedHatAI/Qwen3-8B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 224,
"raw_line": " 72.12 | ",
"row_label_line": 223,
"row_label": "Average Score",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-8B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "suite average of 6 tasks: ARC-Challenge (Acc-Norm, 25-shot); GSM8K (Strict-Match, 5-shot); HellaSwag (Acc-Norm, 10-shot); MMLU (Acc, 5-shot); TruthfulQA (MC2, 0-shot); Winogrande (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-8B-FP8-block@b4fbea2:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-FP8-block",
"artifact_revision": "b4fbea251a7905936ad2656faf78d4ca2839d9ad",
"published_recovery": "100.00"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 block (weights FP8 128x128 blocks, activations FP8 dynamic per-group 128; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.2895,
"quant_score": 0.2912,
"retention": 1.0058721934369603,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-block/blob/b4fbea251a7905936ad2656faf78d4ca2839d9ad/README.md#L269",
"source_id": "quant-redhatai-qwen3-8b-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 269,
"raw_line": " 29.12 | ",
"row_label_line": 267,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "RedHatAI/Qwen3-8B-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-block/README.md",
"sha256": "968a3bf48e1207e72f4a507764246cea71d3d76a5a07cb77ddab22a1a84399c6",
"line": 268,
"raw_line": " 28.95 | ",
"row_label_line": 267,
"row_label": "Average Score",
"row_category": "OpenLLM V2",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"RedHatAI/Qwen3-8B-FP8-block\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Evaluation commands reference the nm-testing copy of this checkpoint; weights were later added to this RedHatAI repository.",
"aggregation": "suite average of 6 tasks: IFEval (Inst Level Strict Acc, 0-shot); BBH (Acc-Norm, 3-shot); Math-Hard (Exact-Match, 4-shot); GPQA (Acc-Norm, 0-shot); MUSR (Acc-Norm, 0-shot); MMLU-Pro (Acc, 5-shot)",
"study": "RedHatAI/Qwen3-8B-FP8-block@b4fbea2:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-8B-FP8-block",
"artifact_revision": "b4fbea251a7905936ad2656faf78d4ca2839d9ad",
"published_recovery": "100.59"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7195,
"quant_score": 0.723,
"retention": 1.004864489228631,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L436",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 436,
"raw_line": " 72.30",
"row_label_line": 432,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 434,
"raw_line": " | 71.95",
"row_label_line": 432,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "100.5%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7597,
"quant_score": 0.8052,
"retention": 1.0598920626563115,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L456",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 456,
"raw_line": " | 80.52",
"row_label_line": 452,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 454,
"raw_line": " | 75.97",
"row_label_line": 452,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "106.0%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6421,
"quant_score": 0.6483,
"retention": 1.009655816850958,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L496",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 496,
"raw_line": " | 64.83",
"row_label_line": 492,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 494,
"raw_line": " | 64.21",
"row_label_line": 492,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "101.0%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3431,
"quant_score": 0.3535,
"retention": 1.0303118624307783,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L568",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 568,
"raw_line": " | 35.35",
"row_label_line": 564,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 566,
"raw_line": " | 34.31",
"row_label_line": 564,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "103.0%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7458,
"quant_score": 0.7635,
"retention": 1.0237329042638776,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L592",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 592,
"raw_line": " | 76.35",
"row_label_line": 588,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 590,
"raw_line": " | 74.58",
"row_label_line": 588,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "102.4%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6521,
"quant_score": 0.6375,
"retention": 0.977610795890201,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L602",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 602,
"raw_line": " | 63.75",
"row_label_line": 598,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 600,
"raw_line": " | 65.21",
"row_label_line": 598,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "97.8%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5859,
"quant_score": 0.6111,
"retention": 1.0430107526881718,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L612",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 612,
"raw_line": " | 61.11",
"row_label_line": 608,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 610,
"raw_line": " | 58.59",
"row_label_line": 608,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "104.3%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "FP8 dynamic (weights FP8 per-channel static, activations FP8 per-token dynamic; llm-compressor QuantizationModifier)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5627,
"quant_score": 0.566,
"retention": 1.0058645814821399,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-FP8-dynamic/blob/05233ce1e0565b5fdc9cfa000ab840152ed30c70/README.md#L632",
"source_id": "quant-redhatai-qwen3-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 632,
"raw_line": " | 56.60",
"row_label_line": 628,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-FP8-dynamic/README.md",
"sha256": "5c836fbce204a5a80f437523400e7b96aa6f6366c6330253e218608767d0ebdd",
"line": 630,
"raw_line": " | 56.27",
"row_label_line": 628,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-FP8-dynamic (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-FP8-dynamic@05233ce:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-FP8-dynamic",
"artifact_revision": "05233ce1e0565b5fdc9cfa000ab840152ed30c70",
"published_recovery": "100.6%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "GSM8K — published row 'gsm8k' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.8726,
"quant_score": 0.8673,
"retention": 0.9939261975704791,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L219",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 219,
"raw_line": " | 86.73 | ",
"row_label_line": 217,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 218,
"raw_line": " 87.26 | ",
"row_label_line": 217,
"row_label": "gsm8k",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "99.39"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "MMLU — published row 'mmlu' (OpenLLM v1 section); lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7497,
"quant_score": 0.7307,
"retention": 0.9746565292783779,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L231",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 231,
"raw_line": " 73.07 | ",
"row_label_line": 229,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 230,
"raw_line": " 74.97 | ",
"row_label_line": 229,
"row_label": "mmlu",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "97.47"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "OpenLLM v1 average of 6 tasks (arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn)",
"full": 0.7159,
"quant_score": 0.7043,
"retention": 0.9837966196396145,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L249",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 249,
"raw_line": " 70.43 | ",
"row_label_line": 247,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 248,
"raw_line": " 71.59 | ",
"row_label_line": 247,
"row_label": "Average",
"row_category": "OpenLLM V1",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: arc_challenge; gsm8k; hellaswag; mmlu; truthfulqa_mc2; winogrande",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "98.38"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "OpenLLM v2 average of 6 tasks (BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)) as published; lm-evaluation-harness (neuralmagic fork) + vLLM (backend vllm, max_model_len 4096, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.4822,
"quant_score": 0.4742,
"retention": 0.9834093737038574,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L292",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 292,
"raw_line": " 47.42 | ",
"row_label_line": 290,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 291,
"raw_line": " 48.22 | ",
"row_label_line": 290,
"row_label": "Average",
"row_category": "OpenLLM V2",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty.",
"aggregation": "suite average of 6 tasks: BBH (3-shot); MMLU-Pro (5-shot); MuSR (0-shot); IFEval (0-shot); GPQA (0-shot); Math-|v|-5 (4-shot)",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "98.33"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "AIME 2024 — published row 'AIME24 (0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.7586,
"quant_score": 0.6207,
"retention": 0.8182177695755339,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L306",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 306,
"raw_line": " 62.07 | ",
"row_label_line": 304,
"row_label": "AIME24 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 305,
"raw_line": " 75.86 | ",
"row_label_line": 304,
"row_label": "AIME24 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "81.82"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "AIME 2025 — published row 'AIME25 (0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.6552,
"quant_score": 0.6207,
"retention": 0.9473443223443224,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L312",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 312,
"raw_line": " 62.07 | ",
"row_label_line": 310,
"row_label": "AIME25 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 311,
"raw_line": " 65.52 | ",
"row_label_line": 310,
"row_label": "AIME25 (0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "94.74"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "GPQA Diamond — published row 'GPQA (Diamond, 0-shot)' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.599,
"quant_score": 0.5482,
"retention": 0.9151919866444074,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L318",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 318,
"raw_line": " 54.82 | ",
"row_label_line": 316,
"row_label": "GPQA (Diamond, 0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 317,
"raw_line": " 59.90 | ",
"row_label_line": 316,
"row_label": "GPQA (Diamond, 0-shot)",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "91.51"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 W4A4 (FP4 weights and activations, group 16; llm-compressor SmoothQuant+QuantizationModifier)",
"metric": "Reasoning average of 3 tasks (AIME24 (0-shot); AIME25 (0-shot); GPQA (Diamond, 0-shot)) as published; lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768, seed 42",
"full": 0.6709,
"quant_score": 0.5965,
"retention": 0.8891041884036369,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-NVFP4/blob/e391349c110709b87bfc2ad2fde3f50dc5839fd8/README.md#L324",
"source_id": "quant-redhatai-qwen3-8b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 324,
"raw_line": " 59.65 | ",
"row_label_line": 322,
"row_label": "Average",
"row_category": "Reasoning",
"column_header": "Qwen3-8B-NVFP4 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-NVFP4/README.md",
"sha256": "8ef59c9fb5c07b1d5b0ab13093366ab70d3c82081c530a5f7cc7d6d8f3ef97fa",
"line": 323,
"raw_line": " 67.09 | ",
"row_label_line": 322,
"row_label": "Average",
"row_category": "Reasoning",
"column_header": "Qwen/Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-NVFP4 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model; lighteval reasoning run; the card does not state enable_thinking.",
"aggregation": "suite average of 3 tasks: AIME24 (0-shot); AIME25 (0-shot); GPQA (Diamond, 0-shot)",
"study": "RedHatAI/Qwen3-8B-NVFP4@e391349:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-NVFP4",
"artifact_revision": "e391349c110709b87bfc2ad2fde3f50dc5839fd8",
"published_recovery": "89.36"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "MMLU — published row 'MMLU (5-shot)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7195,
"quant_score": 0.6974,
"retention": 0.9692842251563584,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L249",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 249,
"raw_line": " 69.74",
"row_label_line": 245,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 247,
"raw_line": " | 71.95",
"row_label_line": 245,
"row_label": "MMLU (5-shot)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "96.9%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "GSM8K — published row 'GSM-8K (5-shot, strict-match)' (OpenLLM v1 section); lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.7597,
"quant_score": 0.7862,
"retention": 1.0348821903382914,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L269",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 269,
"raw_line": " | 78.62",
"row_label_line": 265,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 267,
"raw_line": " | 75.97",
"row_label_line": 265,
"row_label": "GSM-8K (5-shot, strict-match)",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "103.5%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 8192, chat template, few-shot as multiturn)",
"full": 0.6421,
"quant_score": 0.6464,
"retention": 1.0066967762030838,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L309",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 309,
"raw_line": " | 64.64",
"row_label_line": 305,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 307,
"raw_line": " | 64.21",
"row_label_line": 305,
"row_label": "Average",
"row_category": "OpenLLM v1",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "suite average of 6 tasks: MMLU (5-shot); ARC Challenge (25-shot); GSM-8K (5-shot, strict-match); Hellaswag (10-shot); Winogrande (5-shot); TruthfulQA (0-shot, mc2)",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "100.7%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)) as published; lm-evaluation-harness + vLLM (backend vllm, max_model_len 16384, chat template, few-shot as multiturn); per-task scale (raw vs Open LLM Leaderboard-normalized) not stated in card",
"full": 0.3431,
"quant_score": 0.3217,
"retention": 0.9376275138443603,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L381",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 381,
"raw_line": " | 32.17",
"row_label_line": 377,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 379,
"raw_line": " | 34.31",
"row_label_line": 377,
"row_label": "Average",
"row_category": "OpenLLM v2",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "suite average of 6 tasks: MMLU-Pro (5-shot); IFEval (0-shot); BBH (3-shot); Math-lvl-5 (4-shot); GPQA (0-shot); MuSR (0-shot)",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "93.8%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2024 — published row 'AIME 2024' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.7458,
"quant_score": 0.7417,
"retention": 0.9945025475998928,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L405",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 405,
"raw_line": " | 74.17",
"row_label_line": 401,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 403,
"raw_line": " | 74.58",
"row_label_line": 401,
"row_label": "AIME 2024",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "99.5%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "AIME 2025 — published row 'AIME 2025' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.6521,
"quant_score": 0.6198,
"retention": 0.9504677196748966,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L415",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 415,
"raw_line": " | 61.98",
"row_label_line": 411,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 413,
"raw_line": " | 65.21",
"row_label_line": 411,
"row_label": "AIME 2025",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "95.1%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "GPQA Diamond — published row 'GPQA diamond' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5859,
"quant_score": 0.5556,
"retention": 0.9482846902201739,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L425",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 425,
"raw_line": " | 55.56",
"row_label_line": 421,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 423,
"raw_line": " | 58.59",
"row_label_line": 421,
"row_label": "GPQA diamond",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "94.8%"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "INT4 W4A16 (weight-only INT4, asymmetric, group 128; GPTQ, llm-compressor)",
"metric": "LiveCodeBench — published row 'LiveCodeBench' (Reasoning section); lighteval + vLLM, temperature 0.6, top_p 0.95, top_k 20, max_new_tokens 32768",
"full": 0.5627,
"quant_score": 0.5229,
"retention": 0.9292695930335881,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-8B, evaluated by Red Hat in the same card table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-8B-quantized.w4a16/blob/32527053243a382bc56e941c964fc516a5014d39/README.md#L445",
"source_id": "quant-redhatai-qwen3-8b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 445,
"raw_line": " | 52.29",
"row_label_line": 441,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-8B-quantized.w4a16/README.md",
"sha256": "d80c75f0b4f0e1aedee06feff5ad9117cfbb79b0838c12560e9c84ca3af2225d",
"line": 443,
"raw_line": " | 56.27",
"row_label_line": 441,
"row_label": "LiveCodeBench",
"row_category": "Reasoning (generation)",
"column_header": "Qwen3-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) evaluation published in the quantized-model card; the unquantized column is Red Hat's own run in the same table (card states vLLM/lm-eval were used for the evaluations; these baseline values do not appear in the Qwen vendor card), not vendor-reported scores. Evaluated artifact column: \"Qwen3-8B-quantized.w4a16 (this model)\". Relative within-study evidence only; do not compare absolute scores across cards or pool into a universal penalty. Qwen3 hybrid model with its default chat template; the card does not state enable_thinking, so the reasoning mode is not asserted. Card prose says \"asymmetric per-group, group size 64\" but config.json at the pinned revision says symmetric=False, group_size=128; quant label follows config.json (weights were uploaded once, before the README).",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-8B-quantized.w4a16@3252705:Reasoning",
"artifact": "RedHatAI/Qwen3-8B-quantized.w4a16",
"artifact_revision": "32527053243a382bc56e941c964fc516a5014d39",
"published_recovery": "92.9%"
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head and MoE router gates unquantized",
"metric": "OpenLLM v1 average (6 tasks: MMLU 5-shot, ARC-Challenge 25-shot, GSM8K 5-shot strict-match, HellaSwag 10-shot, Winogrande 5-shot, TruthfulQA mc2 0-shot; lm-evaluation-harness, vLLM)",
"full": 0.7375,
"quant_score": 0.7376,
"retention": 1.000135593220339,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-FP8-dynamic/blob/627b08fd72c21c3e0565d87764adce2d87427afc/README.md#L221",
"source_id": "quant-redhatai-qwen3-235b-a22b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/README.md",
"sha256": "ca6dcc1b4793a0421521b9e4c31ec88202cb228e6964a73976edddb2493a3d38",
"line": 221,
"raw_line": " | 73.76",
"row_label_line": 217,
"row_label": "Average",
"column_header": "Qwen3-235B-A22B-FP8-dynamic (this model)",
"table_category": "OpenLLM v1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/README.md",
"sha256": "ca6dcc1b4793a0421521b9e4c31ec88202cb228e6964a73976edddb2493a3d38",
"line": 219,
"raw_line": " | 73.75",
"row_label_line": 217,
"row_label": "Average",
"column_header": "Qwen3-235B-A22B",
"table_category": "OpenLLM v1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-FP8-dynamic model card; baseline column \"Qwen3-235B-A22B\" and quantized column \"Qwen3-235B-A22B-FP8-dynamic (this model)\" are from the same card table. Red Hat evaluation: lm-evaluation-harness --tasks openllm with --apply_chat_template --fewshot_as_multiturn, vLLM backend, max_model_len 8192, tensor_parallel_size 4 (per card). Thinking-mode setting not stated. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: MMLU, ARC-Challenge, GSM8K, HellaSwag, Winogrande, TruthfulQA",
"study": "RedHatAI/Qwen3-235B-A22B-FP8-dynamic@627b08f:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-FP8-dynamic",
"artifact_revision": "627b08fd72c21c3e0565d87764adce2d87427afc",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/config.json",
"sha256": "d0ab025502d25dce66fbdb663eecbc7a6fff6f622ebc1912cfc0e6eeaf09ecc1",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-FP8-dynamic/blob/627b08fd72c21c3e0565d87764adce2d87427afc/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head and MoE router gates unquantized",
"metric": "MMLU 5-shot (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8477,
"quant_score": 0.8461,
"retention": 0.9981125398136133,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-FP8-dynamic/blob/627b08fd72c21c3e0565d87764adce2d87427afc/README.md#L161",
"source_id": "quant-redhatai-qwen3-235b-a22b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/README.md",
"sha256": "ca6dcc1b4793a0421521b9e4c31ec88202cb228e6964a73976edddb2493a3d38",
"line": 161,
"raw_line": " | 84.61",
"row_label_line": 157,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen3-235B-A22B-FP8-dynamic (this model)",
"table_category": "OpenLLM v1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/README.md",
"sha256": "ca6dcc1b4793a0421521b9e4c31ec88202cb228e6964a73976edddb2493a3d38",
"line": 159,
"raw_line": " | 84.77",
"row_label_line": 157,
"row_label": "MMLU (5-shot)",
"column_header": "Qwen3-235B-A22B",
"table_category": "OpenLLM v1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-FP8-dynamic model card; baseline column \"Qwen3-235B-A22B\" and quantized column \"Qwen3-235B-A22B-FP8-dynamic (this model)\" are from the same card table. Red Hat evaluation: lm-evaluation-harness --tasks openllm with --apply_chat_template --fewshot_as_multiturn, vLLM backend, max_model_len 8192, tensor_parallel_size 4 (per card). Thinking-mode setting not stated. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-FP8-dynamic@627b08f:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-FP8-dynamic",
"artifact_revision": "627b08fd72c21c3e0565d87764adce2d87427afc",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/config.json",
"sha256": "d0ab025502d25dce66fbdb663eecbc7a6fff6f622ebc1912cfc0e6eeaf09ecc1",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-FP8-dynamic/blob/627b08fd72c21c3e0565d87764adce2d87427afc/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head and MoE router gates unquantized",
"metric": "GSM8K 5-shot strict-match (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.7422,
"quant_score": 0.7498,
"retention": 1.0102398275397468,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-FP8-dynamic/blob/627b08fd72c21c3e0565d87764adce2d87427afc/README.md#L181",
"source_id": "quant-redhatai-qwen3-235b-a22b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/README.md",
"sha256": "ca6dcc1b4793a0421521b9e4c31ec88202cb228e6964a73976edddb2493a3d38",
"line": 181,
"raw_line": " | 74.98",
"row_label_line": 177,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen3-235B-A22B-FP8-dynamic (this model)",
"table_category": "OpenLLM v1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/README.md",
"sha256": "ca6dcc1b4793a0421521b9e4c31ec88202cb228e6964a73976edddb2493a3d38",
"line": 179,
"raw_line": " | 74.22",
"row_label_line": 177,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "Qwen3-235B-A22B",
"table_category": "OpenLLM v1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-FP8-dynamic model card; baseline column \"Qwen3-235B-A22B\" and quantized column \"Qwen3-235B-A22B-FP8-dynamic (this model)\" are from the same card table. Red Hat evaluation: lm-evaluation-harness --tasks openllm with --apply_chat_template --fewshot_as_multiturn, vLLM backend, max_model_len 8192, tensor_parallel_size 4 (per card). Thinking-mode setting not stated. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-FP8-dynamic@627b08f:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-FP8-dynamic",
"artifact_revision": "627b08fd72c21c3e0565d87764adce2d87427afc",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-FP8-dynamic/config.json",
"sha256": "d0ab025502d25dce66fbdb663eecbc7a6fff6f622ebc1912cfc0e6eeaf09ecc1",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-FP8-dynamic/blob/627b08fd72c21c3e0565d87764adce2d87427afc/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "OpenLLM v1 average (6 tasks: arc_challenge, gsm8k, hellaswag, mmlu, truthfulqa_mc2, winogrande; lm-evaluation-harness (neuralmagic fork), vLLM)",
"full": 0.805,
"quant_score": 0.7997,
"retention": 0.9934161490683229,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L231",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 231,
"raw_line": " | 79.97 | ",
"row_label_line": 229,
"row_label": "Average",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 230,
"raw_line": " 80.50 | ",
"row_label_line": 229,
"row_label": "Average",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: arc_challenge, gsm8k, hellaswag, mmlu, truthfulqa_mc2, winogrande",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "GSM8K (OpenLLM v1 configuration, 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.8992,
"quant_score": 0.9037,
"retention": 1.0050044483985765,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L201",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 201,
"raw_line": " 90.37 | ",
"row_label_line": 199,
"row_label": "gsm8k",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 200,
"raw_line": " 89.92 | ",
"row_label_line": 199,
"row_label": "gsm8k",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "MMLU (OpenLLM v1 configuration, 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.8748,
"quant_score": 0.8708,
"retention": 0.9954275262917238,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L213",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 213,
"raw_line": " 87.08 | ",
"row_label_line": 211,
"row_label": "mmlu",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 212,
"raw_line": " 87.48 | ",
"row_label_line": 211,
"row_label": "mmlu",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "OpenLLM v2 average as published (5 tasks listed: BBH, MMLU-Pro, MuSR, IFEval, GPQA; Math-Hard not included; lm-evaluation-harness, vLLM)",
"full": 0.6033,
"quant_score": 0.6093,
"retention": 1.0099453008453505,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L269",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 269,
"raw_line": " 60.93 | ",
"row_label_line": 267,
"row_label": "Average",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 268,
"raw_line": " 60.33 | ",
"row_label_line": 267,
"row_label": "Average",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 5 tasks: BBH, MMLU-Pro, MuSR, IFEval, GPQA (leaderboard)",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "MMLU-Pro (OpenLLM v2 leaderboard configuration, 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.6351,
"quant_score": 0.6333,
"retention": 0.9971658006613131,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L245",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 245,
"raw_line": " 63.33 | ",
"row_label_line": 243,
"row_label": "MMLU-Pro",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 244,
"raw_line": " 63.51 | ",
"row_label_line": 243,
"row_label": "MMLU-Pro",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "IFEval (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.9017,
"quant_score": 0.9101,
"retention": 1.009315736941333,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L257",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 257,
"raw_line": " 91.01 | ",
"row_label_line": 255,
"row_label": "IFEval",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 256,
"raw_line": " 90.17 | ",
"row_label_line": 255,
"row_label": "IFEval",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "Reasoning average (2 tasks: GPQA Diamond 0-shot, MATH-500 0-shot; lighteval, vLLM)",
"full": 0.817,
"quant_score": 0.7693,
"retention": 0.9416156670746635,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L288",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 288,
"raw_line": " 76.93 | ",
"row_label_line": 286,
"row_label": "Average",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 287,
"raw_line": " 81.70 | ",
"row_label_line": 286,
"row_label": "Average",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 2 tasks: GPQA Diamond, MATH-500",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:Reasoning",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM)",
"full": 0.6919,
"quant_score": 0.6465,
"retention": 0.934383581442405,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L276",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 276,
"raw_line": " 64.65 | ",
"row_label_line": 274,
"row_label": "GPQA (Diamond, 0-shot)",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 275,
"raw_line": " 69.19 | ",
"row_label_line": 274,
"row_label": "GPQA (Diamond, 0-shot)",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:Reasoning",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM)",
"full": 0.942,
"quant_score": 0.892,
"retention": 0.9469214437367305,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/README.md#L282",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 282,
"raw_line": " 89.20 | ",
"row_label_line": 280,
"row_label": "Math-500 (0-shot)",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/README.md",
"sha256": "e2df1815af96e1330f93f149007b4a70a8db95b7a73ae1321d7ae266ef6b5fc6",
"line": 281,
"raw_line": " 94.20 | ",
"row_label_line": 280,
"row_label": "Math-500 (0-shot)",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Card prints OpenLLM v1 average recovery as 101.43 although its own averages give 79.97/80.50 = 0.993; recomputed ratio used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4@9514aa6:Reasoning",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4",
"artifact_revision": "9514aa67dfafa8d459c0372d167597bc7b5811d3",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-NVFP4/config.json",
"sha256": "8563229c147ff2a5b1ea22b95bca7c671e3cb1782cdc55c73079478a07d56575",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4/blob/9514aa67dfafa8d459c0372d167597bc7b5811d3/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "OpenLLM v1 average (6 tasks: arc_challenge, gsm8k, hellaswag, mmlu, truthfulqa_mc2, winogrande; lm-evaluation-harness (neuralmagic fork), vLLM)",
"full": 0.7884,
"quant_score": 0.7874,
"retention": 0.9987316083206494,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L231",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 231,
"raw_line": " 78.74 | ",
"row_label_line": 229,
"row_label": "Average",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 230,
"raw_line": " 78.84 | ",
"row_label_line": 229,
"row_label": "Average",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: arc_challenge, gsm8k, hellaswag, mmlu, truthfulqa_mc2, winogrande",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "GSM8K (OpenLLM v1 configuration, 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.8514,
"quant_score": 0.8643,
"retention": 1.0151515151515151,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L201",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 201,
"raw_line": " 86.43 | ",
"row_label_line": 199,
"row_label": "gsm8k",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 200,
"raw_line": " 85.14 | ",
"row_label_line": 199,
"row_label": "gsm8k",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "MMLU (OpenLLM v1 configuration, 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.8636,
"quant_score": 0.8576,
"retention": 0.9930523390458545,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L213",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 213,
"raw_line": " 85.76 | ",
"row_label_line": 211,
"row_label": "mmlu",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 212,
"raw_line": " 86.36 | ",
"row_label_line": 211,
"row_label": "mmlu",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "OpenLLM v2 average as published (5 tasks listed: BBH, MMLU-Pro, MuSR, IFEval, GPQA; Math-Hard not included; lm-evaluation-harness, vLLM)",
"full": 0.5654,
"quant_score": 0.564,
"retention": 0.9975238769013087,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L269",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 269,
"raw_line": " 56.40 | ",
"row_label_line": 267,
"row_label": "Average",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 268,
"raw_line": " 56.54 | ",
"row_label_line": 267,
"row_label": "Average",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 5 tasks: BBH, MMLU-Pro, MuSR, IFEval, GPQA (leaderboard)",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "MMLU-Pro (OpenLLM v2 leaderboard configuration, 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.5823,
"quant_score": 0.5799,
"retention": 0.9958784131890777,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L245",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 245,
"raw_line": " 57.99 | ",
"row_label_line": 243,
"row_label": "MMLU-Pro",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 244,
"raw_line": " 58.23 | ",
"row_label_line": 243,
"row_label": "MMLU-Pro",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "IFEval (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.8825,
"quant_score": 0.8825,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L257",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 257,
"raw_line": " 88.25 | ",
"row_label_line": 255,
"row_label": "IFEval",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 256,
"raw_line": " 88.25 | ",
"row_label_line": 255,
"row_label": "IFEval",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "Reasoning average (2 tasks: GPQA Diamond 0-shot, MATH-500 0-shot; lighteval, vLLM)",
"full": 0.8361,
"quant_score": 0.817,
"retention": 0.9771558426025595,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L288",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 288,
"raw_line": " 81.70 | ",
"row_label_line": 286,
"row_label": "Average",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 287,
"raw_line": " 83.61 | ",
"row_label_line": 286,
"row_label": "Average",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 2 tasks: GPQA Diamond, MATH-500",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:Reasoning",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM)",
"full": 0.7222,
"quant_score": 0.6919,
"retention": 0.958044862918859,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L276",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 276,
"raw_line": " 69.19 | ",
"row_label_line": 274,
"row_label": "GPQA (Diamond, 0-shot)",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 275,
"raw_line": " 72.22 | ",
"row_label_line": 274,
"row_label": "GPQA (Diamond, 0-shot)",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:Reasoning",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, self-attention projections and MoE router gates left unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM)",
"full": 0.95,
"quant_score": 0.942,
"retention": 0.991578947368421,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/README.md#L282",
"source_id": "quant-redhatai-qwen3-235b-a22b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 282,
"raw_line": " 94.20 | ",
"row_label_line": 280,
"row_label": "Math-500 (0-shot)",
"column_header": "RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/README.md",
"sha256": "42f6f4b1fe49814fc61964364acd77a3b3f449f9d0060c89669953bf6e1894c2",
"line": 281,
"raw_line": " 95.00 | ",
"row_label_line": 280,
"row_label": "Math-500 (0-shot)",
"column_header": "Qwen/Qwen3-235B-A22B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-NVFP4 model card; baseline column \"Qwen/Qwen3-235B-A22B\" and quantized column \"RedHatAI/Qwen3-235B-A22B-NVFP4 (this model)\" are from the same card table. Red Hat evaluation: OpenLLM v1/v2 via lm-evaluation-harness (neuralmagic fork); Reasoning via lighteval (neuralmagic fork); vLLM (per card). Card prints Recovery columns; retention recomputed here. HumanEval_64 pass@2 row not used (outside selected benchmarks). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-NVFP4@1b32bc2:Reasoning",
"artifact": "RedHatAI/Qwen3-235B-A22B-NVFP4",
"artifact_revision": "1b32bc2de6af4465acc7cbafa9d842a15d07457a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-NVFP4/config.json",
"sha256": "b3e61a381ec5db8713456515b3f6b9621cbd65cc51735057fb6c41837c0bea69",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-NVFP4/blob/1b32bc2de6af4465acc7cbafa9d842a15d07457a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ per card text; config.json at pinned revision confirms 8-bit int weights/activations; llm-compressor, compressed-tensors); MoE router gates unquantized",
"metric": "GSM8K-Platinum 5-shot CoT (lm-evaluation-harness gsm8k_platinum_cot_llama, chat completions via vLLM)",
"full": 0.9746,
"quant_score": 0.9699,
"retention": 0.9951775087215268,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/README.md#L223",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 223,
"raw_line": "| GSM8k Platinum (5-shot) | 97.46 | 96.99 | 99.52 |",
"row_label": "GSM8k Platinum (5-shot)",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 223,
"raw_line": "| GSM8k Platinum (5-shot) | 97.46 | 96.99 | 99.52 |",
"row_label": "GSM8k Platinum (5-shot)",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server (max-model-len 262144); lm-evaluation-harness local-chat-completions with chat template, sampling temperature 0.6, top_p 0.95, top_k 20, max_gen_toks 64000; lighteval for MATH-500 (per card). Card creation snippet shows a 4-bit group-128 GPTQ config (apparent copy/paste), but card text and pinned config.json both specify INT8 W8A8; label follows config.json. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8@3ac6f8c:card table",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8",
"artifact_revision": "3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/config.json",
"sha256": "500bd344b3a0c248663c340553ee3a792cc632ea845929757540489b3976c69d",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ per card text; config.json at pinned revision confirms 8-bit int weights/activations; llm-compressor, compressed-tensors); MoE router gates unquantized",
"metric": "MMLU 5-shot CoT (lm-evaluation-harness mmlu_cot_llama, chat completions via vLLM)",
"full": 0.8932,
"quant_score": 0.8927,
"retention": 0.9994402149574564,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/README.md#L224",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 224,
"raw_line": "| MMLU-Cot (5-shot) | 89.32 | 89.27 | 99.94 |",
"row_label": "MMLU-Cot (5-shot)",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 224,
"raw_line": "| MMLU-Cot (5-shot) | 89.32 | 89.27 | 99.94 |",
"row_label": "MMLU-Cot (5-shot)",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server (max-model-len 262144); lm-evaluation-harness local-chat-completions with chat template, sampling temperature 0.6, top_p 0.95, top_k 20, max_gen_toks 64000; lighteval for MATH-500 (per card). Card creation snippet shows a 4-bit group-128 GPTQ config (apparent copy/paste), but card text and pinned config.json both specify INT8 W8A8; label follows config.json. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8@3ac6f8c:card table",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8",
"artifact_revision": "3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/config.json",
"sha256": "500bd344b3a0c248663c340553ee3a792cc632ea845929757540489b3976c69d",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ per card text; config.json at pinned revision confirms 8-bit int weights/activations; llm-compressor, compressed-tensors); MoE router gates unquantized",
"metric": "MMLU-Pro 5-shot (lm-evaluation-harness mmlu_pro_chat, chat completions via vLLM)",
"full": 0.8307,
"quant_score": 0.8343,
"retention": 1.0043336944745396,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/README.md#L225",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 225,
"raw_line": "| MMLU-Pro (5-shot) | 83.07 | 83.43 | 100.43 |",
"row_label": "MMLU-Pro (5-shot)",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 225,
"raw_line": "| MMLU-Pro (5-shot) | 83.07 | 83.43 | 100.43 |",
"row_label": "MMLU-Pro (5-shot)",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server (max-model-len 262144); lm-evaluation-harness local-chat-completions with chat template, sampling temperature 0.6, top_p 0.95, top_k 20, max_gen_toks 64000; lighteval for MATH-500 (per card). Card creation snippet shows a 4-bit group-128 GPTQ config (apparent copy/paste), but card text and pinned config.json both specify INT8 W8A8; label follows config.json. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8@3ac6f8c:card table",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8",
"artifact_revision": "3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/config.json",
"sha256": "500bd344b3a0c248663c340553ee3a792cc632ea845929757540489b3976c69d",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ per card text; config.json at pinned revision confirms 8-bit int weights/activations; llm-compressor, compressed-tensors); MoE router gates unquantized",
"metric": "IFEval (lm-evaluation-harness ifeval, chat completions via vLLM; reported metric variant not stated)",
"full": 0.9169,
"quant_score": 0.9169,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/README.md#L226",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 226,
"raw_line": "| IfEval | 91.69 | 91.69 | 100.00 |",
"row_label": "IfEval",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 226,
"raw_line": "| IfEval | 91.69 | 91.69 | 100.00 |",
"row_label": "IfEval",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server (max-model-len 262144); lm-evaluation-harness local-chat-completions with chat template, sampling temperature 0.6, top_p 0.95, top_k 20, max_gen_toks 64000; lighteval for MATH-500 (per card). Card creation snippet shows a 4-bit group-128 GPTQ config (apparent copy/paste), but card text and pinned config.json both specify INT8 W8A8; label follows config.json. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8@3ac6f8c:card table",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8",
"artifact_revision": "3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/config.json",
"sha256": "500bd344b3a0c248663c340553ee3a792cc632ea845929757540489b3976c69d",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-235B-A22B-Instruct-2507",
"model_id": "qwen3-235b-a22b-instruct-2507",
"params_b": 235,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ per card text; config.json at pinned revision confirms 8-bit int weights/activations; llm-compressor, compressed-tensors); MoE router gates unquantized",
"metric": "MATH-500 0-shot (lighteval math_500 via litellm endpoint, vLLM)",
"full": 0.89,
"quant_score": 0.8967,
"retention": 1.0075280898876404,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-235B-A22B-Instruct-2507, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/README.md#L227",
"source_id": "quant-redhatai-qwen3-235b-a22b-instruct-2507-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 227,
"raw_line": "| Math 500 | 89.00 | 89.67 | 100.75 |",
"row_label": "Math 500",
"column_header": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/README.md",
"sha256": "4d6f69c5f07fb788d999a90c8763dd8454ac38fd005af22715d84d1908503c0b",
"line": 227,
"raw_line": "| Math 500 | 89.00 | 89.67 | 100.75 |",
"row_label": "Math 500",
"column_header": "Qwen/Qwen3-235B-A22B-Instruct-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8 model card; baseline column \"Qwen/Qwen3-235B-A22B-Instruct-2507\" and quantized column \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server (max-model-len 262144); lm-evaluation-harness local-chat-completions with chat template, sampling temperature 0.6, top_p 0.95, top_k 20, max_gen_toks 64000; lighteval for MATH-500 (per card). Card creation snippet shows a 4-bit group-128 GPTQ config (apparent copy/paste), but card text and pinned config.json both specify INT8 W8A8; label follows config.json. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8@3ac6f8c:card table",
"artifact": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8",
"artifact_revision": "3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/config.json",
"sha256": "500bd344b3a0c248663c340553ee3a792cc632ea845929757540489b3976c69d",
"url": "https://huggingface.co/RedHatAI/Qwen3-235B-A22B-Instruct-2507-quantized.w8a8/blob/3ac6f8cbd777cd3c5f855ae1d01fb416b050e66b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v1 average (6 tasks: ARC-Challenge 25-shot acc_norm, GSM8K 5-shot strict-match, HellaSwag 10-shot acc_norm, MMLU 5-shot, TruthfulQA mc2 0-shot, Winogrande 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.6911,
"quant_score": 0.6942,
"retention": 1.0044856026624223,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/README.md#L196",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 196,
"raw_line": "| | **Average Score** | **69.11** | **69.42** | **100.44** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 196,
"raw_line": "| | **Average Score** | **69.11** | **69.42** | **100.44** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: ARC-Challenge, GSM8K, HellaSwag, MMLU, TruthfulQA, Winogrande",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block@ab944c4:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"artifact_revision": "ab944c4c176d211bfa625e8645d12282906761f9",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/config.json",
"sha256": "677e95013908f4322557675dda569146ad6a3b82a866c69273097d51ce8a19b4",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GSM8K 5-shot strict-match (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.3101,
"quant_score": 0.3199,
"retention": 1.031602708803612,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/README.md#L191",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 191,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 31.01 | 31.99 | 103.18 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 191,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 31.01 | 31.99 | 103.18 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block@ab944c4:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"artifact_revision": "ab944c4c176d211bfa625e8645d12282906761f9",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/config.json",
"sha256": "677e95013908f4322557675dda569146ad6a3b82a866c69273097d51ce8a19b4",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU 5-shot acc (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8556,
"quant_score": 0.8555,
"retention": 0.9998831229546518,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/README.md#L193",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 193,
"raw_line": "| | MMLU (Acc, 5-shot) | 85.56 | 85.55 | 99.98 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 193,
"raw_line": "| | MMLU (Acc, 5-shot) | 85.56 | 85.55 | 99.98 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block@ab944c4:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"artifact_revision": "ab944c4c176d211bfa625e8645d12282906761f9",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/config.json",
"sha256": "677e95013908f4322557675dda569146ad6a3b82a866c69273097d51ce8a19b4",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v2 average (6 tasks: IFEval inst-level strict, BBH 3-shot, Math-Hard 4-shot, GPQA 0-shot, MuSR 0-shot, MMLU-Pro 5-shot; lm-evaluation-harness leaderboard, vLLM)",
"full": 0.5704,
"quant_score": 0.5747,
"retention": 1.0075385694249648,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/README.md#L203",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 203,
"raw_line": "| | **Average Score** | **57.04** | **57.47** | **100.75** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 203,
"raw_line": "| | **Average Score** | **57.04** | **57.47** | **100.75** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: IFEval, BBH, Math-Hard, GPQA, MuSR, MMLU-Pro",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block@ab944c4:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"artifact_revision": "ab944c4c176d211bfa625e8645d12282906761f9",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/config.json",
"sha256": "677e95013908f4322557675dda569146ad6a3b82a866c69273097d51ce8a19b4",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "IFEval inst-level strict acc 0-shot (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.9041,
"quant_score": 0.9053,
"retention": 1.001327286804557,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/README.md#L197",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 197,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 90.41 | 90.53 | 100.13 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 197,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 90.41 | 90.53 | 100.13 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block@ab944c4:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"artifact_revision": "ab944c4c176d211bfa625e8645d12282906761f9",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/config.json",
"sha256": "677e95013908f4322557675dda569146ad6a3b82a866c69273097d51ce8a19b4",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU-Pro 5-shot acc (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.5973,
"quant_score": 0.5967,
"retention": 0.998995479658463,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/README.md#L202",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 202,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 59.73 | 59.67 | 99.90 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/README.md",
"sha256": "871e7db61531c0317e3a7b671ddd60a9d242c1c660a80ca96ae02552215e9054",
"line": 202,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 59.73 | 59.67 | 99.90 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block@ab944c4:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block",
"artifact_revision": "ab944c4c176d211bfa625e8645d12282906761f9",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-block/config.json",
"sha256": "677e95013908f4322557675dda569146ad6a3b82a866c69273097d51ce8a19b4",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block/blob/ab944c4c176d211bfa625e8645d12282906761f9/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v1 average (6 tasks: ARC-Challenge 25-shot acc_norm, GSM8K 5-shot strict-match, HellaSwag 10-shot acc_norm, MMLU 5-shot, TruthfulQA mc2 0-shot, Winogrande 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.6911,
"quant_score": 0.6956,
"retention": 1.0065113587035162,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/README.md#L198",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 198,
"raw_line": "| | **Average Score** | **69.11** | **69.56** | **100.65** |",
"row_label": "Average Score",
"column_header": "Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 198,
"raw_line": "| | **Average Score** | **69.11** | **69.56** | **100.65** |",
"row_label": "Average Score",
"column_header": "Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: ARC-Challenge, GSM8K, HellaSwag, MMLU, TruthfulQA, Winogrande",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic@fd5b264:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"artifact_revision": "fd5b264444ce2bb0686d4914965bae94e890f422",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/config.json",
"sha256": "0382e1d5a624116302fddba45b613e2988ecb7db8d34a62be0b4b5dbb21ce1c3",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GSM8K 5-shot strict-match (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.3101,
"quant_score": 0.3321,
"retention": 1.070944856497904,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/README.md#L193",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 193,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 31.01 | 33.21 | 107.09 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 193,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 31.01 | 33.21 | 107.09 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic@fd5b264:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"artifact_revision": "fd5b264444ce2bb0686d4914965bae94e890f422",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/config.json",
"sha256": "0382e1d5a624116302fddba45b613e2988ecb7db8d34a62be0b4b5dbb21ce1c3",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU 5-shot acc (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8556,
"quant_score": 0.854,
"retention": 0.9981299672744273,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/README.md#L195",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 195,
"raw_line": "| | MMLU (Acc, 5-shot) | 85.56 | 85.40 | 99.81 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 195,
"raw_line": "| | MMLU (Acc, 5-shot) | 85.56 | 85.40 | 99.81 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic@fd5b264:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"artifact_revision": "fd5b264444ce2bb0686d4914965bae94e890f422",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/config.json",
"sha256": "0382e1d5a624116302fddba45b613e2988ecb7db8d34a62be0b4b5dbb21ce1c3",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v2 average (6 tasks: IFEval inst-level strict, BBH 3-shot, Math-Hard 4-shot, GPQA 0-shot, MuSR 0-shot, MMLU-Pro 5-shot; lm-evaluation-harness leaderboard, vLLM)",
"full": 0.5704,
"quant_score": 0.5727,
"retention": 1.004032258064516,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/README.md#L205",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 205,
"raw_line": "| | **Average Score** | **57.04** | **57.27** | **100.40** |",
"row_label": "Average Score",
"column_header": "Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 205,
"raw_line": "| | **Average Score** | **57.04** | **57.27** | **100.40** |",
"row_label": "Average Score",
"column_header": "Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: IFEval, BBH, Math-Hard, GPQA, MuSR, MMLU-Pro",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic@fd5b264:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"artifact_revision": "fd5b264444ce2bb0686d4914965bae94e890f422",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/config.json",
"sha256": "0382e1d5a624116302fddba45b613e2988ecb7db8d34a62be0b4b5dbb21ce1c3",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "IFEval inst-level strict acc 0-shot (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.9041,
"quant_score": 0.9029,
"retention": 0.998672713195443,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/README.md#L199",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 199,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 90.41 | 90.29 | 99.87 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 199,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 90.41 | 90.29 | 99.87 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic@fd5b264:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"artifact_revision": "fd5b264444ce2bb0686d4914965bae94e890f422",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/config.json",
"sha256": "0382e1d5a624116302fddba45b613e2988ecb7db8d34a62be0b4b5dbb21ce1c3",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU-Pro 5-shot acc (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.5973,
"quant_score": 0.6001,
"retention": 1.0046877615938388,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/README.md#L204",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 204,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 59.73 | 60.01 | 100.46 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/README.md",
"sha256": "b77724ab759bb7c0493d5d214a18bfcf04d5b1627d408e56a5f6d422506c5d77",
"line": 204,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 59.73 | 60.01 | 100.46 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. OpenLLM v1 was run without a chat template, which explains the low absolute GSM8K for an instruct model; use only as within-table retention. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic@fd5b264:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic",
"artifact_revision": "fd5b264444ce2bb0686d4914965bae94e890f422",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/config.json",
"sha256": "0382e1d5a624116302fddba45b613e2988ecb7db8d34a62be0b4b5dbb21ce1c3",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic/blob/fd5b264444ce2bb0686d4914965bae94e890f422/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v1 average (6 tasks: ARC-Challenge 25-shot acc_norm, GSM8K 5-shot strict-match, HellaSwag 10-shot acc_norm, MMLU 5-shot, TruthfulQA mc2 0-shot, Winogrande 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.7389,
"quant_score": 0.7376,
"retention": 0.9982406279604819,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/README.md#L287",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 287,
"raw_line": "| | **Average Score** | **73.89** | **73.76** | **99.82** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 287,
"raw_line": "| | **Average Score** | **73.89** | **73.76** | **99.82** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16\" are from the same card table. Red Hat evaluation (per card commands): lm-evaluation-harness --tasks openllm and --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn, vLLM, tensor_parallel_size 1, max_model_len 15000. This card's BF16 baseline differs from the FP8 cards' baseline (different run/configuration: e.g. HellaSwag 63.90 vs 83.25); compare only within this table. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: ARC-Challenge, GSM8K, HellaSwag, MMLU, TruthfulQA, Winogrande",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16@ac9dc5b:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"artifact_revision": "ac9dc5b939ba948ab378b8638cfcce4ac4d5642b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/config.json",
"sha256": "58153d27af24e8b3636f5d590a8ed809a4a987dc081fc459a1c6241272a4bd47",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GSM8K 5-shot strict-match (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8158,
"quant_score": 0.8218,
"retention": 1.0073547438097572,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/README.md#L282",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 282,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 81.58 | 82.18 | 100.74 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 282,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 81.58 | 82.18 | 100.74 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16\" are from the same card table. Red Hat evaluation (per card commands): lm-evaluation-harness --tasks openllm and --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn, vLLM, tensor_parallel_size 1, max_model_len 15000. This card's BF16 baseline differs from the FP8 cards' baseline (different run/configuration: e.g. HellaSwag 63.90 vs 83.25); compare only within this table. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16@ac9dc5b:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"artifact_revision": "ac9dc5b939ba948ab378b8638cfcce4ac4d5642b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/config.json",
"sha256": "58153d27af24e8b3636f5d590a8ed809a4a987dc081fc459a1c6241272a4bd47",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU 5-shot acc (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8556,
"quant_score": 0.8503,
"retention": 0.9938055165965404,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/README.md#L284",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 284,
"raw_line": "| | MMLU (Acc, 5-shot) | 85.56 | 85.03 | 99.38 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 284,
"raw_line": "| | MMLU (Acc, 5-shot) | 85.56 | 85.03 | 99.38 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16\" are from the same card table. Red Hat evaluation (per card commands): lm-evaluation-harness --tasks openllm and --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn, vLLM, tensor_parallel_size 1, max_model_len 15000. This card's BF16 baseline differs from the FP8 cards' baseline (different run/configuration: e.g. HellaSwag 63.90 vs 83.25); compare only within this table. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16@ac9dc5b:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"artifact_revision": "ac9dc5b939ba948ab378b8638cfcce4ac4d5642b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/config.json",
"sha256": "58153d27af24e8b3636f5d590a8ed809a4a987dc081fc459a1c6241272a4bd47",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v2 average (6 tasks: IFEval inst-level strict, BBH 3-shot, Math-Hard 4-shot, GPQA 0-shot, MuSR 0-shot, MMLU-Pro 5-shot; lm-evaluation-harness leaderboard, vLLM)",
"full": 0.5765,
"quant_score": 0.5793,
"retention": 1.0048568950563748,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/README.md#L294",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 294,
"raw_line": "| | **Average Score** | **57.65** | **57.93** | **100.49** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 294,
"raw_line": "| | **Average Score** | **57.65** | **57.93** | **100.49** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16\" are from the same card table. Red Hat evaluation (per card commands): lm-evaluation-harness --tasks openllm and --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn, vLLM, tensor_parallel_size 1, max_model_len 15000. This card's BF16 baseline differs from the FP8 cards' baseline (different run/configuration: e.g. HellaSwag 63.90 vs 83.25); compare only within this table. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: IFEval, BBH, Math-Hard, GPQA, MuSR, MMLU-Pro",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16@ac9dc5b:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"artifact_revision": "ac9dc5b939ba948ab378b8638cfcce4ac4d5642b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/config.json",
"sha256": "58153d27af24e8b3636f5d590a8ed809a4a987dc081fc459a1c6241272a4bd47",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "IFEval inst-level strict acc 0-shot (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.7746,
"quant_score": 0.807,
"retention": 1.0418280402788538,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/README.md#L288",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 288,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 77.46 | 80.70 | 104.18 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 288,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 77.46 | 80.70 | 104.18 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16\" are from the same card table. Red Hat evaluation (per card commands): lm-evaluation-harness --tasks openllm and --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn, vLLM, tensor_parallel_size 1, max_model_len 15000. This card's BF16 baseline differs from the FP8 cards' baseline (different run/configuration: e.g. HellaSwag 63.90 vs 83.25); compare only within this table. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16@ac9dc5b:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"artifact_revision": "ac9dc5b939ba948ab378b8638cfcce4ac4d5642b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/config.json",
"sha256": "58153d27af24e8b3636f5d590a8ed809a4a987dc081fc459a1c6241272a4bd47",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU-Pro 5-shot acc (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.7635,
"quant_score": 0.7548,
"retention": 0.9886051080550099,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/README.md#L293",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 293,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 76.35 | 75.48 | 98.86 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/README.md",
"sha256": "edcef24fd00350a24d4dab1da12f74312f9fc91fa44eddeb732dbc2664594d6e",
"line": 293,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 76.35 | 75.48 | 98.86 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Instruct\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16\" are from the same card table. Red Hat evaluation (per card commands): lm-evaluation-harness --tasks openllm and --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn, vLLM, tensor_parallel_size 1, max_model_len 15000. This card's BF16 baseline differs from the FP8 cards' baseline (different run/configuration: e.g. HellaSwag 63.90 vs 83.25); compare only within this table. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16@ac9dc5b:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16",
"artifact_revision": "ac9dc5b939ba948ab378b8638cfcce4ac4d5642b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/config.json",
"sha256": "58153d27af24e8b3636f5d590a8ed809a4a987dc081fc459a1c6241272a4bd47",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w4a16/blob/ac9dc5b939ba948ab378b8638cfcce4ac4d5642b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (QuantizationModifier W8A8, Open-Platypus calibration); lm_head, MoE router/shared-expert gates and linear-attention layers unquantized",
"metric": "AIME 2025 0-shot (lighteval aime25, litellm endpoint, vLLM; mean of 6 seeded repetitions)",
"full": 0.6278,
"quant_score": 0.65,
"retention": 1.0353615801210576,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/README.md#L267",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 267,
"raw_line": " 65.00",
"row_label_line": 263,
"row_label": "AIME25",
"column_header": "Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 265,
"raw_line": " | 62.78",
"row_label_line": 263,
"row_label": "AIME25",
"column_header": "Qwen3-Next-80B-A3B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)\" are from the same card table. Red Hat evaluation: vLLM server (TP=2); lm-evaluation-harness local-chat-completions (temperature 0.7, top_p 0.8, top_k 20, max_gen_toks 16000) and lighteval via litellm; results averaged over 6 repetitions with different seeds (per card). The card's lighteval litellm_config.yaml names hosted_vllm/RedHatAI/Phi-4-reasoning-FP8-dynamic (apparent copy/paste); the table itself is labelled for the Qwen3-Next checkpoints. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8@509f3bf:card table",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8",
"artifact_revision": "509f3bf6f13341cf4bbca584acd2ba020d09aaf2",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/config.json",
"sha256": "ddf5155372a8aca24245580578710a44f22d611f6feeb2eb921ee2aa10c32771",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (QuantizationModifier W8A8, Open-Platypus calibration); lm_head, MoE router/shared-expert gates and linear-attention layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval gpqa:diamond; mean of 6 seeded repetitions)",
"full": 0.7458,
"quant_score": 0.7517,
"retention": 1.0079109680879592,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/README.md#L277",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 277,
"raw_line": " | 75.17",
"row_label_line": 273,
"row_label": "GPQA Diamond",
"column_header": "Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 275,
"raw_line": " | 74.58",
"row_label_line": 273,
"row_label": "GPQA Diamond",
"column_header": "Qwen3-Next-80B-A3B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)\" are from the same card table. Red Hat evaluation: vLLM server (TP=2); lm-evaluation-harness local-chat-completions (temperature 0.7, top_p 0.8, top_k 20, max_gen_toks 16000) and lighteval via litellm; results averaged over 6 repetitions with different seeds (per card). The card's lighteval litellm_config.yaml names hosted_vllm/RedHatAI/Phi-4-reasoning-FP8-dynamic (apparent copy/paste); the table itself is labelled for the Qwen3-Next checkpoints. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8@509f3bf:card table",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8",
"artifact_revision": "509f3bf6f13341cf4bbca584acd2ba020d09aaf2",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/config.json",
"sha256": "ddf5155372a8aca24245580578710a44f22d611f6feeb2eb921ee2aa10c32771",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (QuantizationModifier W8A8, Open-Platypus calibration); lm_head, MoE router/shared-expert gates and linear-attention layers unquantized",
"metric": "MATH-500 0-shot (lighteval math_500; mean of 6 seeded repetitions)",
"full": 0.8973,
"quant_score": 0.9057,
"retention": 1.0093614175860917,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/README.md#L287",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 287,
"raw_line": " | 90.57",
"row_label_line": 283,
"row_label": "Math 500",
"column_header": "Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 285,
"raw_line": " | 89.73",
"row_label_line": 283,
"row_label": "Math 500",
"column_header": "Qwen3-Next-80B-A3B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)\" are from the same card table. Red Hat evaluation: vLLM server (TP=2); lm-evaluation-harness local-chat-completions (temperature 0.7, top_p 0.8, top_k 20, max_gen_toks 16000) and lighteval via litellm; results averaged over 6 repetitions with different seeds (per card). The card's lighteval litellm_config.yaml names hosted_vllm/RedHatAI/Phi-4-reasoning-FP8-dynamic (apparent copy/paste); the table itself is labelled for the Qwen3-Next checkpoints. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8@509f3bf:card table",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8",
"artifact_revision": "509f3bf6f13341cf4bbca584acd2ba020d09aaf2",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/config.json",
"sha256": "ddf5155372a8aca24245580578710a44f22d611f6feeb2eb921ee2aa10c32771",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (QuantizationModifier W8A8, Open-Platypus calibration); lm_head, MoE router/shared-expert gates and linear-attention layers unquantized",
"metric": "MMLU-Pro 5-shot (lm-evaluation-harness mmlu_pro_chat, chat completions; mean of 6 seeded repetitions)",
"full": 0.7862,
"quant_score": 0.7885,
"retention": 1.0029254642584584,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/README.md#L297",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 297,
"raw_line": " | 78.85",
"row_label_line": 293,
"row_label": "MMLU-Pro",
"column_header": "Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 295,
"raw_line": " | 78.62",
"row_label_line": 293,
"row_label": "MMLU-Pro",
"column_header": "Qwen3-Next-80B-A3B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)\" are from the same card table. Red Hat evaluation: vLLM server (TP=2); lm-evaluation-harness local-chat-completions (temperature 0.7, top_p 0.8, top_k 20, max_gen_toks 16000) and lighteval via litellm; results averaged over 6 repetitions with different seeds (per card). The card's lighteval litellm_config.yaml names hosted_vllm/RedHatAI/Phi-4-reasoning-FP8-dynamic (apparent copy/paste); the table itself is labelled for the Qwen3-Next checkpoints. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8@509f3bf:card table",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8",
"artifact_revision": "509f3bf6f13341cf4bbca584acd2ba020d09aaf2",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/config.json",
"sha256": "ddf5155372a8aca24245580578710a44f22d611f6feeb2eb921ee2aa10c32771",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (QuantizationModifier W8A8, Open-Platypus calibration); lm_head, MoE router/shared-expert gates and linear-attention layers unquantized",
"metric": "IFEval (lm-evaluation-harness ifeval, chat completions; metric variant not stated; mean of 6 seeded repetitions)",
"full": 0.9145,
"quant_score": 0.9151,
"retention": 1.0006560962274467,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/README.md#L307",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 307,
"raw_line": " | 91.51",
"row_label_line": 303,
"row_label": "IFEval",
"column_header": "Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 305,
"raw_line": " | 91.45",
"row_label_line": 303,
"row_label": "IFEval",
"column_header": "Qwen3-Next-80B-A3B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)\" are from the same card table. Red Hat evaluation: vLLM server (TP=2); lm-evaluation-harness local-chat-completions (temperature 0.7, top_p 0.8, top_k 20, max_gen_toks 16000) and lighteval via litellm; results averaged over 6 repetitions with different seeds (per card). The card's lighteval litellm_config.yaml names hosted_vllm/RedHatAI/Phi-4-reasoning-FP8-dynamic (apparent copy/paste); the table itself is labelled for the Qwen3-Next checkpoints. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8@509f3bf:card table",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8",
"artifact_revision": "509f3bf6f13341cf4bbca584acd2ba020d09aaf2",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/config.json",
"sha256": "ddf5155372a8aca24245580578710a44f22d611f6feeb2eb921ee2aa10c32771",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Instruct",
"model_id": "qwen3-next-80b-a3b-instruct",
"params_b": 80,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (QuantizationModifier W8A8, Open-Platypus calibration); lm_head, MoE router/shared-expert gates and linear-attention layers unquantized",
"metric": "GSM8K 5-shot (lm-evaluation-harness gsm8k, chat completions, fewshot as multiturn; mean of 6 seeded repetitions)",
"full": 0.6971,
"quant_score": 0.6974,
"retention": 1.000430354325061,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Instruct, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/README.md#L317",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-instruct-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 317,
"raw_line": " | 69.74",
"row_label_line": 313,
"row_label": "GSM8k",
"column_header": "Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/README.md",
"sha256": "3654b1ee190c3b9d1bf1df51eb5335da7bc67d6af2198edde97f62556aca79ac",
"line": 315,
"raw_line": " | 69.71",
"row_label_line": 313,
"row_label": "GSM8k",
"column_header": "Qwen3-Next-80B-A3B-Instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 model card; baseline column \"Qwen3-Next-80B-A3B-Instruct\" and quantized column \"Qwen3-Next-80B-A3B-Instruct-quantized.w8a8 (this model)\" are from the same card table. Red Hat evaluation: vLLM server (TP=2); lm-evaluation-harness local-chat-completions (temperature 0.7, top_p 0.8, top_k 20, max_gen_toks 16000) and lighteval via litellm; results averaged over 6 repetitions with different seeds (per card). The card's lighteval litellm_config.yaml names hosted_vllm/RedHatAI/Phi-4-reasoning-FP8-dynamic (apparent copy/paste); the table itself is labelled for the Qwen3-Next checkpoints. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8@509f3bf:card table",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8",
"artifact_revision": "509f3bf6f13341cf4bbca584acd2ba020d09aaf2",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/config.json",
"sha256": "ddf5155372a8aca24245580578710a44f22d611f6feeb2eb921ee2aa10c32771",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Instruct-quantized.w8a8/blob/509f3bf6f13341cf4bbca584acd2ba020d09aaf2/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v1 average (6 tasks: ARC-Challenge 25-shot acc_norm, GSM8K 5-shot strict-match, HellaSwag 10-shot acc_norm, MMLU 5-shot, TruthfulQA mc2 0-shot, Winogrande 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.7319,
"quant_score": 0.7322,
"retention": 1.000409892061757,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L199",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 199,
"raw_line": "| | **Average Score** | **73.19** | **73.22** | **100.04** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 199,
"raw_line": "| | **Average Score** | **73.19** | **73.22** | **100.04** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: ARC-Challenge, GSM8K, HellaSwag, MMLU, TruthfulQA, Winogrande",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GSM8K 5-shot strict-match (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8461,
"quant_score": 0.8476,
"retention": 1.001772840089824,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L194",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 194,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 84.61 | 84.76 | 100.18 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 194,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 84.61 | 84.76 | 100.18 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU 5-shot acc (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8495,
"quant_score": 0.8505,
"retention": 1.0011771630370807,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L196",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 196,
"raw_line": "| | MMLU (Acc, 5-shot) | 84.95 | 85.05 | 100.12 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 196,
"raw_line": "| | MMLU (Acc, 5-shot) | 84.95 | 85.05 | 100.12 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v2 average (6 tasks: IFEval inst-level strict, BBH 3-shot, Math-Hard 4-shot, GPQA 0-shot, MuSR 0-shot, MMLU-Pro 5-shot; lm-evaluation-harness leaderboard, vLLM)",
"full": 0.3905,
"quant_score": 0.384,
"retention": 0.9833546734955185,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L206",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 206,
"raw_line": "| | **Average Score** | **39.05** | **38.40** | **98.34** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 206,
"raw_line": "| | **Average Score** | **39.05** | **38.40** | **98.34** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: IFEval, BBH, Math-Hard, GPQA, MuSR, MMLU-Pro",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "IFEval inst-level strict acc 0-shot (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.4484,
"quant_score": 0.4245,
"retention": 0.9466993755575378,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L200",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 200,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 44.84 | 42.45 | 94.67 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 200,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 44.84 | 42.45 | 94.67 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU-Pro 5-shot acc (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.727,
"quant_score": 0.7277,
"retention": 1.0009628610729024,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L205",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 205,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 72.70 | 72.77 | 100.10 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 205,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 72.70 | 72.77 | 100.10 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "Reasoning average (3 tasks: AIME 2025, MATH-500, GPQA Diamond; pass@1 over n=8 samples; lighteval, vLLM)",
"full": 0.7591,
"quant_score": 0.7052,
"retention": 0.928994862336978,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L210",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 210,
"raw_line": "| | **Average Score** | **75.91** | **70.52** | **92.90** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 210,
"raw_line": "| | **Average Score** | **75.91** | **70.52** | **92.90** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 3 tasks: AIME 2025, MATH-500, GPQA Diamond",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "AIME 2025 pass@1 (n=8; lighteval, vLLM)",
"full": 0.6,
"quant_score": 0.5,
"retention": 0.8333333333333334,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L207",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 207,
"raw_line": "| Reasoning | AIME25 (pass@1, n=8) | 60.00 | 50.00 | 83.33 |",
"row_label": "AIME25 (pass@1, n=8)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 207,
"raw_line": "| Reasoning | AIME25 (pass@1, n=8) | 60.00 | 50.00 | 83.33 |",
"row_label": "AIME25 (pass@1, n=8)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MATH-500 pass@1 (n=8; lighteval, vLLM)",
"full": 0.94,
"quant_score": 0.868,
"retention": 0.9234042553191489,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L208",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 208,
"raw_line": "| | MATH-500 (pass@1, n=8) | 94.00 | 86.80 | 92.34 |",
"row_label": "MATH-500 (pass@1, n=8)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 208,
"raw_line": "| | MATH-500 (pass@1, n=8) | 94.00 | 86.80 | 92.34 |",
"row_label": "MATH-500 (pass@1, n=8)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GPQA Diamond pass@1 (n=8; lighteval, vLLM)",
"full": 0.7374,
"quant_score": 0.7475,
"retention": 1.0136967724437211,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/README.md#L209",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 209,
"raw_line": "| | GPQA-Diamond (pass@1, n=8) | 73.74 | 74.75 | 101.37 |",
"row_label": "GPQA-Diamond (pass@1, n=8)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/README.md",
"sha256": "043bc922ee68280226af4c06cde2e6e9487166675fdde871fd3c961a2a4e3268",
"line": 209,
"raw_line": "| | GPQA-Diamond (pass@1, n=8) | 73.74 | 74.75 | 101.37 |",
"row_label": "GPQA-Diamond (pass@1, n=8)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic\" are from the same card table. Red Hat evaluation (per card commands): OpenLLM v1 via lm-evaluation-harness --tasks openllm WITHOUT chat template (add_bos_token=True); OpenLLM v2 via --tasks leaderboard with --apply_chat_template --fewshot_as_multiturn; vLLM, tensor_parallel_size 2, max_model_len 16384. Reasoning rows: pass@1 over n=8 samples (lighteval). AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic@c67f214:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic",
"artifact_revision": "c67f21402fe030480002fc69f23df9c7cba75bb5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/config.json",
"sha256": "9faea32bc6e467c1cf13133346520243a4845bb02b0f9d2ff034e2a6c65483e1",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-dynamic/blob/c67f21402fe030480002fc69f23df9c7cba75bb5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v1 average (6 tasks: ARC-Challenge 25-shot acc_norm, GSM8K 5-shot strict-match, HellaSwag 10-shot acc_norm, MMLU 5-shot, TruthfulQA mc2 0-shot, Winogrande 5-shot; lm-evaluation-harness, vLLM)",
"full": 0.7319,
"quant_score": 0.7293,
"retention": 0.9964476021314387,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L267",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 267,
"raw_line": "| | **Average Score** | **73.19** | **72.93** | **99.64** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 267,
"raw_line": "| | **Average Score** | **73.19** | **72.93** | **99.64** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: ARC-Challenge, GSM8K, HellaSwag, MMLU, TruthfulQA, Winogrande",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GSM8K 5-shot strict-match (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8461,
"quant_score": 0.8378,
"retention": 0.9901902848363078,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L262",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 262,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 84.61 | 83.78 | 99.02 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 262,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 84.61 | 83.78 | 99.02 |",
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU 5-shot acc (OpenLLM v1 configuration; lm-evaluation-harness, vLLM)",
"full": 0.8495,
"quant_score": 0.8456,
"retention": 0.9954090641553855,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L264",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 264,
"raw_line": "| | MMLU (Acc, 5-shot) | 84.95 | 84.56 | 99.54 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 264,
"raw_line": "| | MMLU (Acc, 5-shot) | 84.95 | 84.56 | 99.54 |",
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:OpenLLM v1",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "OpenLLM v2 average (6 tasks: IFEval inst-level strict, BBH 3-shot, Math-Hard 4-shot, GPQA 0-shot, MuSR 0-shot, MMLU-Pro 5-shot; lm-evaluation-harness leaderboard, vLLM)",
"full": 0.3905,
"quant_score": 0.3885,
"retention": 0.9948783610755442,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L274",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 274,
"raw_line": "| | **Average Score** | **39.05** | **38.85** | **99.49** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 274,
"raw_line": "| | **Average Score** | **39.05** | **38.85** | **99.49** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 6 tasks: IFEval, BBH, Math-Hard, GPQA, MuSR, MMLU-Pro",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "IFEval inst-level strict acc 0-shot (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.4484,
"quant_score": 0.4604,
"retention": 1.0267618198037465,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L268",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 268,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 44.84 | 46.04 | 102.68 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 268,
"raw_line": "| OpenLLM V2 | IFEval (Inst Level Strict Acc, 0-shot) | 44.84 | 46.04 | 102.68 |",
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MMLU-Pro 5-shot acc (OpenLLM v2 leaderboard configuration; lm-evaluation-harness, vLLM)",
"full": 0.727,
"quant_score": 0.7143,
"retention": 0.9825309491059148,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L273",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 273,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 72.70 | 71.43 | 98.25 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 273,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 72.70 | 71.43 | 98.25 |",
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "OpenLLM V2"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:OpenLLM v2",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "Reasoning average (3 tasks: AIME 2025, MATH-500, GPQA Diamond; pass@1 over n=8 samples; lighteval, vLLM)",
"full": 0.7591,
"quant_score": 0.7031,
"retention": 0.926228428402055,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L278",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 278,
"raw_line": "| | **Average Score** | **75.91** | **70.31** | **92.62** |",
"row_label": "Average Score",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 278,
"raw_line": "| | **Average Score** | **75.91** | **70.31** | **92.62** |",
"row_label": "Average Score",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "suite average of 3 tasks: AIME 2025, MATH-500, GPQA Diamond",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "AIME 2025 pass@1 (n=8; lighteval, vLLM)",
"full": 0.6,
"quant_score": 0.5,
"retention": 0.8333333333333334,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L275",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 275,
"raw_line": "| Reasoning | AIME25 (pass@1, n=8) | 60.00 | 50.00 | 83.33 |",
"row_label": "AIME25 (pass@1, n=8)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 275,
"raw_line": "| Reasoning | AIME25 (pass@1, n=8) | 60.00 | 50.00 | 83.33 |",
"row_label": "AIME25 (pass@1, n=8)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MATH-500 pass@1 (n=8; lighteval, vLLM)",
"full": 0.94,
"quant_score": 0.872,
"retention": 0.9276595744680851,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L276",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 276,
"raw_line": "| | MATH-500 (pass@1, n=8) | 94.00 | 87.20 | 92.77 |",
"row_label": "MATH-500 (pass@1, n=8)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 276,
"raw_line": "| | MATH-500 (pass@1, n=8) | 94.00 | 87.20 | 92.77 |",
"row_label": "MATH-500 (pass@1, n=8)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GPQA Diamond pass@1 (n=8; lighteval, vLLM)",
"full": 0.7374,
"quant_score": 0.7374,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/README.md#L277",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 277,
"raw_line": "| | GPQA-Diamond (pass@1, n=8) | 73.74 | 73.74 | 100.00 |",
"row_label": "GPQA-Diamond (pass@1, n=8)",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/README.md",
"sha256": "034e72eca0aac5d3450e0c3ee9c2865286bb8ab9d40d1bcae30a2a414aa03180",
"line": 277,
"raw_line": "| | GPQA-Diamond (pass@1, n=8) | 73.74 | 73.74 | 100.00 |",
"row_label": "GPQA-Diamond (pass@1, n=8)",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16 model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16\" are from the same card table. Red Hat evaluation: card command blocks show lm-evaluation-harness openllm/leaderboard with chat template (vLLM, TP=1, max_model_len 15000) and lighteval reasoning tasks; the BF16 baseline values are identical to the sibling FP8-dynamic card, whose commands differ, so the exact baseline configuration is uncertain. AIME 2025 at n=8 moves in 1/30-question steps per sample; the 60.00 -> 50.00 drop is a small-sample result. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16@76f7498:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16",
"artifact_revision": "76f7498176afa06571650e00dd3f423f608c4369",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/config.json",
"sha256": "424e0c6f47389f84a095884f8d724f05e51cb4cbc815f03755f251a1d18ae9fe",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-quantized.w4a16/blob/76f7498176afa06571650e00dd3f423f608c4369/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "AIME 2025 (reasoning table; per sibling FP8-dynamic card: pass@1 over n=8, lighteval, vLLM)",
"full": 0.6,
"quant_score": 0.5,
"retention": 0.8333333333333334,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/README.md#L194",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 194,
"raw_line": "| AIME 2025 | 60.00 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 |",
"row_label": "AIME 2025",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 194,
"raw_line": "| AIME 2025 | 60.00 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 |",
"row_label": "AIME 2025",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block\" are from the same card table. Red Hat evaluation table comparing Qwen/Qwen3-Next-80B-A3B-Thinking (BF16) with several quantized checkpoints. The card does not describe the reasoning-task protocol; the BF16 and FP8-dynamic/w4a16 values are identical to the sibling Red Hat cards that state pass@1 with n=8 via lighteval. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block@30b4ae8:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block",
"artifact_revision": "30b4ae8ac43a84fa8b328ad8b1c64adb7070867f",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/config.json",
"sha256": "e1981ab413afce1e838959eac1051d2ea5fd5c6b88d994ba508ad47463961b2d",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "MATH-500 (reasoning table; per sibling FP8-dynamic card: pass@1 over n=8, lighteval, vLLM)",
"full": 0.94,
"quant_score": 0.944,
"retention": 1.004255319148936,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/README.md#L195",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 195,
"raw_line": "| MATH-500 | 94.00 | 87.60 | 93.19 | 94.40 | 100.43 | 86.80 | 92.34 | 87.20 | 92.77 |",
"row_label": "MATH-500",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 195,
"raw_line": "| MATH-500 | 94.00 | 87.60 | 93.19 | 94.40 | 100.43 | 86.80 | 92.34 | 87.20 | 92.77 |",
"row_label": "MATH-500",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block\" are from the same card table. Red Hat evaluation table comparing Qwen/Qwen3-Next-80B-A3B-Thinking (BF16) with several quantized checkpoints. The card does not describe the reasoning-task protocol; the BF16 and FP8-dynamic/w4a16 values are identical to the sibling Red Hat cards that state pass@1 with n=8 via lighteval. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block@30b4ae8:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block",
"artifact_revision": "30b4ae8ac43a84fa8b328ad8b1c64adb7070867f",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/config.json",
"sha256": "e1981ab413afce1e838959eac1051d2ea5fd5c6b88d994ba508ad47463961b2d",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 block (E4M3; weights static 128x128 blocks, activations dynamic per-group 128; llm-compressor FP8_BLOCK, compressed-tensors); lm_head, MoE router/shared-expert gates and linear-attention (Gated DeltaNet) layers unquantized",
"metric": "GPQA Diamond (reasoning table; per sibling FP8-dynamic card: pass@1 over n=8, lighteval, vLLM)",
"full": 0.7374,
"quant_score": 0.7677,
"retention": 1.0410903173311636,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/README.md#L196",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 196,
"raw_line": "| GPQA Diamond | 73.74 | 76.77 | 104.11 | 76.77 | 104.11 | 74.75 | 101.37 | 73.74 | 100.00 |",
"row_label": "GPQA Diamond",
"column_header": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 196,
"raw_line": "| GPQA Diamond | 73.74 | 76.77 | 104.11 | 76.77 | 104.11 | 74.75 | 101.37 | 73.74 | 100.00 |",
"row_label": "GPQA Diamond",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block\" are from the same card table. Red Hat evaluation table comparing Qwen/Qwen3-Next-80B-A3B-Thinking (BF16) with several quantized checkpoints. The card does not describe the reasoning-task protocol; the BF16 and FP8-dynamic/w4a16 values are identical to the sibling Red Hat cards that state pass@1 with n=8 via lighteval. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block@30b4ae8:Reasoning",
"artifact": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block",
"artifact_revision": "30b4ae8ac43a84fa8b328ad8b1c64adb7070867f",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/config.json",
"sha256": "e1981ab413afce1e838959eac1051d2ea5fd5c6b88d994ba508ad47463961b2d",
"url": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 (Qwen official checkpoint Qwen/Qwen3-Next-80B-A3B-Thinking-FP8; recipe not described in this card)",
"metric": "AIME 2025 (reasoning table; per sibling FP8-dynamic card: pass@1 over n=8, lighteval, vLLM)",
"full": 0.6,
"quant_score": 0.5,
"retention": 0.8333333333333334,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/README.md#L194",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 194,
"raw_line": "| AIME 2025 | 60.00 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 |",
"row_label": "AIME 2025",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 194,
"raw_line": "| AIME 2025 | 60.00 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 | 50.00 | 83.33 |",
"row_label": "AIME 2025",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"Qwen/Qwen3-Next-80B-A3B-Thinking-FP8\" are from the same card table. Red Hat evaluation table comparing Qwen/Qwen3-Next-80B-A3B-Thinking (BF16) with several quantized checkpoints. The card does not describe the reasoning-task protocol; the BF16 and FP8-dynamic/w4a16 values are identical to the sibling Red Hat cards that state pass@1 with n=8 via lighteval. Evaluated artifact is Qwen's own FP8 release (revision not stated), measured by Red Hat alongside its own quantizations. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block@30b4ae8:Reasoning [Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 column]",
"artifact": "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8",
"artifact_revision": null
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 (Qwen official checkpoint Qwen/Qwen3-Next-80B-A3B-Thinking-FP8; recipe not described in this card)",
"metric": "MATH-500 (reasoning table; per sibling FP8-dynamic card: pass@1 over n=8, lighteval, vLLM)",
"full": 0.94,
"quant_score": 0.876,
"retention": 0.9319148936170213,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/README.md#L195",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 195,
"raw_line": "| MATH-500 | 94.00 | 87.60 | 93.19 | 94.40 | 100.43 | 86.80 | 92.34 | 87.20 | 92.77 |",
"row_label": "MATH-500",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 195,
"raw_line": "| MATH-500 | 94.00 | 87.60 | 93.19 | 94.40 | 100.43 | 86.80 | 92.34 | 87.20 | 92.77 |",
"row_label": "MATH-500",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"Qwen/Qwen3-Next-80B-A3B-Thinking-FP8\" are from the same card table. Red Hat evaluation table comparing Qwen/Qwen3-Next-80B-A3B-Thinking (BF16) with several quantized checkpoints. The card does not describe the reasoning-task protocol; the BF16 and FP8-dynamic/w4a16 values are identical to the sibling Red Hat cards that state pass@1 with n=8 via lighteval. Evaluated artifact is Qwen's own FP8 release (revision not stated), measured by Red Hat alongside its own quantizations. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block@30b4ae8:Reasoning [Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 column]",
"artifact": "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8",
"artifact_revision": null
},
{
"model": "Qwen3-Next-80B-A3B-Thinking",
"model_id": "qwen3-next-80b-a3b-thinking",
"params_b": 80,
"quant": "FP8 (Qwen official checkpoint Qwen/Qwen3-Next-80B-A3B-Thinking-FP8; recipe not described in this card)",
"metric": "GPQA Diamond (reasoning table; per sibling FP8-dynamic card: pass@1 over n=8, lighteval, vLLM)",
"full": 0.7374,
"quant_score": 0.7677,
"retention": 1.0410903173311636,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3-Next-80B-A3B-Thinking, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block/blob/30b4ae8ac43a84fa8b328ad8b1c64adb7070867f/README.md#L196",
"source_id": "quant-redhatai-qwen3-next-80b-a3b-thinking-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 196,
"raw_line": "| GPQA Diamond | 73.74 | 76.77 | 104.11 | 76.77 | 104.11 | 74.75 | 101.37 | 73.74 | 100.00 |",
"row_label": "GPQA Diamond",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3-Next-80B-A3B-Thinking-FP8-block/README.md",
"sha256": "5c5649b99c953f03cbd31879995f583fccc294754877f6c3caee462362dc60d5",
"line": 196,
"raw_line": "| GPQA Diamond | 73.74 | 76.77 | 104.11 | 76.77 | 104.11 | 74.75 | 101.37 | 73.74 | 100.00 |",
"row_label": "GPQA Diamond",
"column_header": "Qwen/Qwen3-Next-80B-A3B-Thinking"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block model card; baseline column \"Qwen/Qwen3-Next-80B-A3B-Thinking\" and quantized column \"Qwen/Qwen3-Next-80B-A3B-Thinking-FP8\" are from the same card table. Red Hat evaluation table comparing Qwen/Qwen3-Next-80B-A3B-Thinking (BF16) with several quantized checkpoints. The card does not describe the reasoning-task protocol; the BF16 and FP8-dynamic/w4a16 values are identical to the sibling Red Hat cards that state pass@1 with n=8 via lighteval. Evaluated artifact is Qwen's own FP8 release (revision not stated), measured by Red Hat alongside its own quantizations. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3-Next-80B-A3B-Thinking-FP8-block@30b4ae8:Reasoning [Qwen/Qwen3-Next-80B-A3B-Thinking-FP8 column]",
"artifact": "Qwen/Qwen3-Next-80B-A3B-Thinking-FP8",
"artifact_revision": null
},
{
"model": "Qwen3.5-122B-A10B",
"model_id": "qwen3-5-122b-a10b",
"params_b": 125,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness gsm8k_platinum_cot_llama, chat completions; mean of 3 seeds)",
"full": 0.9559,
"quant_score": 0.957,
"retention": 1.001150747986191,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-122B-A10B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/README.md#L179",
"source_id": "quant-redhatai-qwen3-5-122b-a10b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 179,
"raw_line": " | 95.70 | ",
"row_label_line": 177,
"row_label": "GSM8K-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 178,
"raw_line": " 95.59 | ",
"row_label_line": 177,
"row_label": "GSM8K-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-122B-A10B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-122B-A10B\" and quantized column \"RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Card prints IFEval prompt-level recovery as 100.0% although 91.13/92.08 = 0.990; recomputed ratio used. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic@dae1750:card table",
"artifact": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"artifact_revision": "dae17507e035bac7e87c3daaf3939656a6550784",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/config.json",
"sha256": "3e6f919d90c48e499013eaedebdc91fc8abf31ebe524f5eeae7fcdecf704bf47",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-122B-A10B",
"model_id": "qwen3-5-122b-a10b",
"params_b": 125,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness mmlu_pro_chat, chat completions; mean of 3 seeds)",
"full": 0.8696,
"quant_score": 0.8696,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-122B-A10B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/README.md#L185",
"source_id": "quant-redhatai-qwen3-5-122b-a10b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 185,
"raw_line": " 86.96 | ",
"row_label_line": 183,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 184,
"raw_line": " 86.96 | ",
"row_label_line": 183,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-122B-A10B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-122B-A10B\" and quantized column \"RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Card prints IFEval prompt-level recovery as 100.0% although 91.13/92.08 = 0.990; recomputed ratio used. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic@dae1750:card table",
"artifact": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"artifact_revision": "dae17507e035bac7e87c3daaf3939656a6550784",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/config.json",
"sha256": "3e6f919d90c48e499013eaedebdc91fc8abf31ebe524f5eeae7fcdecf704bf47",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-122B-A10B",
"model_id": "qwen3-5-122b-a10b",
"params_b": 125,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "MATH-500 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.852,
"quant_score": 0.8513,
"retention": 0.9991784037558685,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-122B-A10B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/README.md#L191",
"source_id": "quant-redhatai-qwen3-5-122b-a10b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 191,
"raw_line": " 85.13 | ",
"row_label_line": 189,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 190,
"raw_line": " 85.20 | ",
"row_label_line": 189,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-122B-A10B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-122B-A10B\" and quantized column \"RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Card prints IFEval prompt-level recovery as 100.0% although 91.13/92.08 = 0.990; recomputed ratio used. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic@dae1750:card table",
"artifact": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"artifact_revision": "dae17507e035bac7e87c3daaf3939656a6550784",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/config.json",
"sha256": "3e6f919d90c48e499013eaedebdc91fc8abf31ebe524f5eeae7fcdecf704bf47",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-122B-A10B",
"model_id": "qwen3-5-122b-a10b",
"params_b": 125,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "AIME 2025 0-shot (lighteval via litellm; mean of 8 seeds)",
"full": 0.9208,
"quant_score": 0.925,
"retention": 1.0045612510860122,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-122B-A10B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/README.md#L197",
"source_id": "quant-redhatai-qwen3-5-122b-a10b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 197,
"raw_line": " 92.50 | ",
"row_label_line": 195,
"row_label": "AIME 25 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 196,
"raw_line": " 92.08 | ",
"row_label_line": 195,
"row_label": "AIME 25 (0-shot)",
"column_header": "Qwen/Qwen3.5-122B-A10B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-122B-A10B\" and quantized column \"RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Card prints IFEval prompt-level recovery as 100.0% although 91.13/92.08 = 0.990; recomputed ratio used. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic@dae1750:card table",
"artifact": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"artifact_revision": "dae17507e035bac7e87c3daaf3939656a6550784",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/config.json",
"sha256": "3e6f919d90c48e499013eaedebdc91fc8abf31ebe524f5eeae7fcdecf704bf47",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-122B-A10B",
"model_id": "qwen3-5-122b-a10b",
"params_b": 125,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8737,
"quant_score": 0.8603,
"retention": 0.9846629277784136,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-122B-A10B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/README.md#L203",
"source_id": "quant-redhatai-qwen3-5-122b-a10b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 203,
"raw_line": " 86.03 | ",
"row_label_line": 201,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 202,
"raw_line": " 87.37 | ",
"row_label_line": 201,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-122B-A10B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-122B-A10B\" and quantized column \"RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Card prints IFEval prompt-level recovery as 100.0% although 91.13/92.08 = 0.990; recomputed ratio used. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic@dae1750:card table",
"artifact": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"artifact_revision": "dae17507e035bac7e87c3daaf3939656a6550784",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/config.json",
"sha256": "3e6f919d90c48e499013eaedebdc91fc8abf31ebe524f5eeae7fcdecf704bf47",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-122B-A10B",
"model_id": "qwen3-5-122b-a10b",
"params_b": 125,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness ifeval, chat completions; mean of 3 seeds)",
"full": 0.9208,
"quant_score": 0.9113,
"retention": 0.9896828844483059,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-122B-A10B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/README.md#L210",
"source_id": "quant-redhatai-qwen3-5-122b-a10b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 210,
"raw_line": " 91.13 | ",
"row_label_line": 208,
"row_label": "IFEval prompt-level strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 209,
"raw_line": " 92.08 | ",
"row_label_line": 208,
"row_label": "IFEval prompt-level strict (0-shot)",
"column_header": "Qwen/Qwen3.5-122B-A10B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-122B-A10B\" and quantized column \"RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Card prints IFEval prompt-level recovery as 100.0% although 91.13/92.08 = 0.990; recomputed ratio used. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic@dae1750:card table",
"artifact": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"artifact_revision": "dae17507e035bac7e87c3daaf3939656a6550784",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/config.json",
"sha256": "3e6f919d90c48e499013eaedebdc91fc8abf31ebe524f5eeae7fcdecf704bf47",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-122B-A10B",
"model_id": "qwen3-5-122b-a10b",
"params_b": 125,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "LiveCodeBench v6 code generation 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8133,
"quant_score": 0.819,
"retention": 1.0070084839542603,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-122B-A10B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/README.md#L223",
"source_id": "quant-redhatai-qwen3-5-122b-a10b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 223,
"raw_line": " 81.90 | ",
"row_label_line": 221,
"row_label": "LiveCodeBench v6 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"table_category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/README.md",
"sha256": "1090744fecadd4528c252b64dcdd62cc172a64da972ed6dc42b41470e852f56a",
"line": 222,
"raw_line": " 81.33 | ",
"row_label_line": 221,
"row_label": "LiveCodeBench v6 (0-shot)",
"column_header": "Qwen/Qwen3.5-122B-A10B",
"table_category": "Coding"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-122B-A10B\" and quantized column \"RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Card prints IFEval prompt-level recovery as 100.0% although 91.13/92.08 = 0.990; recomputed ratio used. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic@dae1750:card table",
"artifact": "RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic",
"artifact_revision": "dae17507e035bac7e87c3daaf3939656a6550784",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-122B-A10B-FP8-dynamic/config.json",
"sha256": "3e6f919d90c48e499013eaedebdc91fc8abf31ebe524f5eeae7fcdecf704bf47",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-122B-A10B-FP8-dynamic/blob/dae17507e035bac7e87c3daaf3939656a6550784/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-35B-A3B",
"model_id": "qwen3-5-35b-a3b",
"params_b": 36,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness gsm8k_platinum_cot_llama, chat completions; mean of 3 seeds)",
"full": 0.9498,
"quant_score": 0.9512,
"retention": 1.0014739945251632,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/README.md#L187",
"source_id": "quant-redhatai-qwen3-5-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 187,
"raw_line": " 95.12 | ",
"row_label_line": 185,
"row_label": "GSM8K-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 186,
"raw_line": " 94.98 | ",
"row_label_line": 185,
"row_label": "GSM8K-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-35B-A3B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-35B-A3B\" and quantized column \"RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic@5448394:card table",
"artifact": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"artifact_revision": "5448394de7a7f81eb5e92f9608aa91a14bd11ceb",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/config.json",
"sha256": "dba2fc8960ae3d03922c8ecd97c477ddc2560f65b867ea70e7a683df8efe568a",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-35B-A3B",
"model_id": "qwen3-5-35b-a3b",
"params_b": 36,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness mmlu_pro_chat, chat completions; mean of 3 seeds)",
"full": 0.8565,
"quant_score": 0.8565,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/README.md#L193",
"source_id": "quant-redhatai-qwen3-5-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 193,
"raw_line": " 85.65 | ",
"row_label_line": 191,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 192,
"raw_line": " 85.65 | ",
"row_label_line": 191,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-35B-A3B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-35B-A3B\" and quantized column \"RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic@5448394:card table",
"artifact": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"artifact_revision": "5448394de7a7f81eb5e92f9608aa91a14bd11ceb",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/config.json",
"sha256": "dba2fc8960ae3d03922c8ecd97c477ddc2560f65b867ea70e7a683df8efe568a",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-35B-A3B",
"model_id": "qwen3-5-35b-a3b",
"params_b": 36,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "MATH-500 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8493,
"quant_score": 0.8433,
"retention": 0.9929353585305546,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/README.md#L199",
"source_id": "quant-redhatai-qwen3-5-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 199,
"raw_line": " 84.33 | ",
"row_label_line": 197,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 198,
"raw_line": " 84.93 | ",
"row_label_line": 197,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-35B-A3B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-35B-A3B\" and quantized column \"RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic@5448394:card table",
"artifact": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"artifact_revision": "5448394de7a7f81eb5e92f9608aa91a14bd11ceb",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/config.json",
"sha256": "dba2fc8960ae3d03922c8ecd97c477ddc2560f65b867ea70e7a683df8efe568a",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-35B-A3B",
"model_id": "qwen3-5-35b-a3b",
"params_b": 36,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "AIME 2025 0-shot (lighteval via litellm; mean of 8 seeds)",
"full": 0.9125,
"quant_score": 0.9125,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/README.md#L205",
"source_id": "quant-redhatai-qwen3-5-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 205,
"raw_line": " 91.25 | ",
"row_label_line": 203,
"row_label": "AIME 25 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 204,
"raw_line": " 91.25 | ",
"row_label_line": 203,
"row_label": "AIME 25 (0-shot)",
"column_header": "Qwen/Qwen3.5-35B-A3B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-35B-A3B\" and quantized column \"RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic@5448394:card table",
"artifact": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"artifact_revision": "5448394de7a7f81eb5e92f9608aa91a14bd11ceb",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/config.json",
"sha256": "dba2fc8960ae3d03922c8ecd97c477ddc2560f65b867ea70e7a683df8efe568a",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-35B-A3B",
"model_id": "qwen3-5-35b-a3b",
"params_b": 36,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.83,
"quant_score": 0.8316,
"retention": 1.0019277108433735,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/README.md#L211",
"source_id": "quant-redhatai-qwen3-5-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 211,
"raw_line": " 83.16 | ",
"row_label_line": 209,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 210,
"raw_line": " 83.00 | ",
"row_label_line": 209,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-35B-A3B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-35B-A3B\" and quantized column \"RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic@5448394:card table",
"artifact": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"artifact_revision": "5448394de7a7f81eb5e92f9608aa91a14bd11ceb",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/config.json",
"sha256": "dba2fc8960ae3d03922c8ecd97c477ddc2560f65b867ea70e7a683df8efe568a",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-35B-A3B",
"model_id": "qwen3-5-35b-a3b",
"params_b": 36,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness ifeval, chat completions; mean of 3 seeds)",
"full": 0.91,
"quant_score": 0.9045,
"retention": 0.9939560439560439,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/README.md#L218",
"source_id": "quant-redhatai-qwen3-5-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 218,
"raw_line": " 90.45 | ",
"row_label_line": 216,
"row_label": "IFEval prompt-level strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 217,
"raw_line": " 91.00 | ",
"row_label_line": 216,
"row_label": "IFEval prompt-level strict (0-shot)",
"column_header": "Qwen/Qwen3.5-35B-A3B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-35B-A3B\" and quantized column \"RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic@5448394:card table",
"artifact": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"artifact_revision": "5448394de7a7f81eb5e92f9608aa91a14bd11ceb",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/config.json",
"sha256": "dba2fc8960ae3d03922c8ecd97c477ddc2560f65b867ea70e7a683df8efe568a",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-35B-A3B",
"model_id": "qwen3-5-35b-a3b",
"params_b": 36,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision-encoder layers unquantized",
"metric": "LiveCodeBench v6 code generation 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.7543,
"quant_score": 0.7638,
"retention": 1.0125944584382873,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/README.md#L231",
"source_id": "quant-redhatai-qwen3-5-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 231,
"raw_line": " 76.38 | ",
"row_label_line": 229,
"row_label": "LiveCodeBench v6 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"table_category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/README.md",
"sha256": "61cf60dab8193ff8d8de3c9d669bb2a20863126c1587b4f13587e2d0abbef468",
"line": 230,
"raw_line": " 75.43 | ",
"row_label_line": 229,
"row_label": "LiveCodeBench v6 (0-shot)",
"column_header": "Qwen/Qwen3.5-35B-A3B",
"table_category": "Coding"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-35B-A3B\" and quantized column \"RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic@5448394:card table",
"artifact": "RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic",
"artifact_revision": "5448394de7a7f81eb5e92f9608aa91a14bd11ceb",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-35B-A3B-FP8-dynamic/config.json",
"sha256": "dba2fc8960ae3d03922c8ecd97c477ddc2560f65b867ea70e7a683df8efe568a",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-35B-A3B-FP8-dynamic/blob/5448394de7a7f81eb5e92f9608aa91a14bd11ceb/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-397B-A17B",
"model_id": "qwen3-5-397b-a17b",
"params_b": 397,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness gsm8k_platinum_cot_llama, chat completions; mean of 3 seeds)",
"full": 0.9553,
"quant_score": 0.9537,
"retention": 0.9983251334659269,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-397B-A17B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/README.md#L187",
"source_id": "quant-redhatai-qwen3-5-397b-a17b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 187,
"raw_line": " 95.37 | ",
"row_label_line": 185,
"row_label": "GSM8K-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 186,
"raw_line": " 95.53 | ",
"row_label_line": 185,
"row_label": "GSM8K-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-397B-A17B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-397B-A17B\" and quantized column \"RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Tensor parallel size 4. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic@b752046:card table",
"artifact": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"artifact_revision": "b75204601a4140423d5107fdccd6b928cfb1d7c5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/config.json",
"sha256": "047c40c62d22c47d14065b65732a18c6a5e4b1fde455c54239ad989e0319c378",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-397B-A17B",
"model_id": "qwen3-5-397b-a17b",
"params_b": 397,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness mmlu_pro_chat, chat completions; mean of 3 seeds)",
"full": 0.8827,
"quant_score": 0.8833,
"retention": 1.0006797326384955,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-397B-A17B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/README.md#L193",
"source_id": "quant-redhatai-qwen3-5-397b-a17b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 193,
"raw_line": " 88.33 | ",
"row_label_line": 191,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 192,
"raw_line": " 88.27 | ",
"row_label_line": 191,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-397B-A17B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-397B-A17B\" and quantized column \"RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Tensor parallel size 4. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic@b752046:card table",
"artifact": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"artifact_revision": "b75204601a4140423d5107fdccd6b928cfb1d7c5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/config.json",
"sha256": "047c40c62d22c47d14065b65732a18c6a5e4b1fde455c54239ad989e0319c378",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-397B-A17B",
"model_id": "qwen3-5-397b-a17b",
"params_b": 397,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8493,
"quant_score": 0.8453,
"retention": 0.9952902390203697,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-397B-A17B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/README.md#L199",
"source_id": "quant-redhatai-qwen3-5-397b-a17b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 199,
"raw_line": " 84.53 | ",
"row_label_line": 197,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 198,
"raw_line": " 84.93 | ",
"row_label_line": 197,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-397B-A17B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-397B-A17B\" and quantized column \"RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Tensor parallel size 4. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic@b752046:card table",
"artifact": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"artifact_revision": "b75204601a4140423d5107fdccd6b928cfb1d7c5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/config.json",
"sha256": "047c40c62d22c47d14065b65732a18c6a5e4b1fde455c54239ad989e0319c378",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-397B-A17B",
"model_id": "qwen3-5-397b-a17b",
"params_b": 397,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval via litellm; mean of 8 seeds)",
"full": 0.9292,
"quant_score": 0.9458,
"retention": 1.017864829961257,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-397B-A17B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/README.md#L205",
"source_id": "quant-redhatai-qwen3-5-397b-a17b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 205,
"raw_line": " 94.58 | ",
"row_label_line": 203,
"row_label": "AIME 25 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 204,
"raw_line": " 92.92 | ",
"row_label_line": 203,
"row_label": "AIME 25 (0-shot)",
"column_header": "Qwen/Qwen3.5-397B-A17B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-397B-A17B\" and quantized column \"RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Tensor parallel size 4. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic@b752046:card table",
"artifact": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"artifact_revision": "b75204601a4140423d5107fdccd6b928cfb1d7c5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/config.json",
"sha256": "047c40c62d22c47d14065b65732a18c6a5e4b1fde455c54239ad989e0319c378",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-397B-A17B",
"model_id": "qwen3-5-397b-a17b",
"params_b": 397,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8973,
"quant_score": 0.8872,
"retention": 0.9887440098071993,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-397B-A17B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/README.md#L211",
"source_id": "quant-redhatai-qwen3-5-397b-a17b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 211,
"raw_line": " 88.72 | ",
"row_label_line": 209,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 210,
"raw_line": " 89.73 | ",
"row_label_line": 209,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-397B-A17B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-397B-A17B\" and quantized column \"RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Tensor parallel size 4. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic@b752046:card table",
"artifact": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"artifact_revision": "b75204601a4140423d5107fdccd6b928cfb1d7c5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/config.json",
"sha256": "047c40c62d22c47d14065b65732a18c6a5e4b1fde455c54239ad989e0319c378",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-397B-A17B",
"model_id": "qwen3-5-397b-a17b",
"params_b": 397,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness ifeval, chat completions; mean of 3 seeds)",
"full": 0.9063,
"quant_score": 0.907,
"retention": 1.0007723711795211,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-397B-A17B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/README.md#L218",
"source_id": "quant-redhatai-qwen3-5-397b-a17b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 218,
"raw_line": " 90.70 | ",
"row_label_line": 216,
"row_label": "IFEval prompt-level strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 217,
"raw_line": " 90.63 | ",
"row_label_line": 216,
"row_label": "IFEval prompt-level strict (0-shot)",
"column_header": "Qwen/Qwen3.5-397B-A17B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-397B-A17B\" and quantized column \"RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Tensor parallel size 4. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic@b752046:card table",
"artifact": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"artifact_revision": "b75204601a4140423d5107fdccd6b928cfb1d7c5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/config.json",
"sha256": "047c40c62d22c47d14065b65732a18c6a5e4b1fde455c54239ad989e0319c378",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-397B-A17B",
"model_id": "qwen3-5-397b-a17b",
"params_b": 397,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "LiveCodeBench v6 code generation 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8381,
"quant_score": 0.8362,
"retention": 0.9977329674263216,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-397B-A17B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/README.md#L231",
"source_id": "quant-redhatai-qwen3-5-397b-a17b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 231,
"raw_line": " 83.62 | ",
"row_label_line": 229,
"row_label": "LiveCodeBench v6 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"table_category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/README.md",
"sha256": "46268d8afc7f5cecdbe67a16d91989bb076a162c93d6ad9d52e8bf47c6d75012",
"line": 230,
"raw_line": " 83.81 | ",
"row_label_line": 229,
"row_label": "LiveCodeBench v6 (0-shot)",
"column_header": "Qwen/Qwen3.5-397B-A17B",
"table_category": "Coding"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-397B-A17B\" and quantized column \"RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); sampling temperature 0.6, top_p 0.95, top_k 20, presence_penalty 0.0, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per card). Tensor parallel size 4. IFEval inst-level row not recorded separately (same benchmark, correlated). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic@b752046:card table",
"artifact": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic",
"artifact_revision": "b75204601a4140423d5107fdccd6b928cfb1d7c5",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-397B-A17B-FP8-dynamic/config.json",
"sha256": "047c40c62d22c47d14065b65732a18c6a5e4b1fde455c54239ad989e0319c378",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic/blob/b75204601a4140423d5107fdccd6b928cfb1d7c5/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.942,
"quant_score": 0.945,
"retention": 1.0031847133757963,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/README.md#L161",
"source_id": "quant-redhatai-qwen3-5-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 161,
"raw_line": " 94.5% | ",
"row_label_line": 159,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 160,
"raw_line": " 94.2% | ",
"row_label_line": 159,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-FP8-dynamic@397b7ba:card table",
"artifact": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"artifact_revision": "397b7ba47a99b3221ebfc0cfc5a279118cb733ad",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/config.json",
"sha256": "adb21629f2577f579147b5d1fe3c2a90c7495c02bef2d9a1637fdc007731c4a8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.793,
"quant_score": 0.791,
"retention": 0.9974779319041615,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/README.md#L167",
"source_id": "quant-redhatai-qwen3-5-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 167,
"raw_line": " 79.1% | ",
"row_label_line": 165,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 166,
"raw_line": " 79.3% | ",
"row_label_line": 165,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-FP8-dynamic@397b7ba:card table",
"artifact": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"artifact_revision": "397b7ba47a99b3221ebfc0cfc5a279118cb733ad",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/config.json",
"sha256": "adb21629f2577f579147b5d1fe3c2a90c7495c02bef2d9a1637fdc007731c4a8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.88,
"quant_score": 0.884,
"retention": 1.0045454545454546,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/README.md#L173",
"source_id": "quant-redhatai-qwen3-5-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 173,
"raw_line": " 88.4% | ",
"row_label_line": 171,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 172,
"raw_line": " 88.0% | ",
"row_label_line": 171,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-FP8-dynamic@397b7ba:card table",
"artifact": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"artifact_revision": "397b7ba47a99b3221ebfc0cfc5a279118cb733ad",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/config.json",
"sha256": "adb21629f2577f579147b5d1fe3c2a90c7495c02bef2d9a1637fdc007731c4a8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.846,
"quant_score": 0.847,
"retention": 1.0011820330969268,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/README.md#L186",
"source_id": "quant-redhatai-qwen3-5-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 186,
"raw_line": " 84.7% | ",
"row_label_line": 184,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 185,
"raw_line": " 84.6% | ",
"row_label_line": 184,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-FP8-dynamic@397b7ba:card table",
"artifact": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"artifact_revision": "397b7ba47a99b3221ebfc0cfc5a279118cb733ad",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/config.json",
"sha256": "adb21629f2577f579147b5d1fe3c2a90c7495c02bef2d9a1637fdc007731c4a8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval, vLLM; mean of 8 seeds)",
"full": 0.85,
"quant_score": 0.85,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/README.md#L192",
"source_id": "quant-redhatai-qwen3-5-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 192,
"raw_line": " 85.0% | ",
"row_label_line": 190,
"row_label": "AIME 2025 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 191,
"raw_line": " 85.0% | ",
"row_label_line": 190,
"row_label": "AIME 2025 (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-FP8-dynamic@397b7ba:card table",
"artifact": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"artifact_revision": "397b7ba47a99b3221ebfc0cfc5a279118cb733ad",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/config.json",
"sha256": "adb21629f2577f579147b5d1fe3c2a90c7495c02bef2d9a1637fdc007731c4a8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.768,
"quant_score": 0.763,
"retention": 0.9934895833333334,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/README.md#L198",
"source_id": "quant-redhatai-qwen3-5-4b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 198,
"raw_line": " 76.3% | ",
"row_label_line": 196,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/README.md",
"sha256": "3380110f73da6682f5fc35e5377721d9151cdcab3610f76a5f8f3d8039e0b5d8",
"line": 197,
"raw_line": " 76.8% | ",
"row_label_line": 196,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-FP8-dynamic@397b7ba:card table",
"artifact": "RedHatAI/Qwen3.5-4B-FP8-dynamic",
"artifact_revision": "397b7ba47a99b3221ebfc0cfc5a279118cb733ad",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-FP8-dynamic/config.json",
"sha256": "adb21629f2577f579147b5d1fe3c2a90c7495c02bef2d9a1637fdc007731c4a8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-FP8-dynamic/blob/397b7ba47a99b3221ebfc0cfc5a279118cb733ad/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.945,
"quant_score": 0.945,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/README.md#L180",
"source_id": "quant-redhatai-qwen3-5-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 180,
"raw_line": " 94.5% | ",
"row_label_line": 178,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 179,
"raw_line": " 94.5% | ",
"row_label_line": 178,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w4a16@7a61387:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"artifact_revision": "7a613872f394578b0b52b683ff4ac47516b4bcaf",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/config.json",
"sha256": "c0e62b3248e5f8847c76ea8f18e19c52db6d1cbcb4f2e636cee33063dd541043",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.793,
"quant_score": 0.775,
"retention": 0.9773013871374527,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/README.md#L186",
"source_id": "quant-redhatai-qwen3-5-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 186,
"raw_line": " 77.5% | ",
"row_label_line": 184,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 185,
"raw_line": " 79.3% | ",
"row_label_line": 184,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w4a16@7a61387:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"artifact_revision": "7a613872f394578b0b52b683ff4ac47516b4bcaf",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/config.json",
"sha256": "c0e62b3248e5f8847c76ea8f18e19c52db6d1cbcb4f2e636cee33063dd541043",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.883,
"quant_score": 0.861,
"retention": 0.9750849377123443,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/README.md#L192",
"source_id": "quant-redhatai-qwen3-5-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 192,
"raw_line": " 86.1% | ",
"row_label_line": 190,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 191,
"raw_line": " 88.3% | ",
"row_label_line": 190,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w4a16@7a61387:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"artifact_revision": "7a613872f394578b0b52b683ff4ac47516b4bcaf",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/config.json",
"sha256": "c0e62b3248e5f8847c76ea8f18e19c52db6d1cbcb4f2e636cee33063dd541043",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.845,
"quant_score": 0.837,
"retention": 0.9905325443786982,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/README.md#L205",
"source_id": "quant-redhatai-qwen3-5-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 205,
"raw_line": " 83.7% | ",
"row_label_line": 203,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 204,
"raw_line": " 84.5% | ",
"row_label_line": 203,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w4a16@7a61387:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"artifact_revision": "7a613872f394578b0b52b683ff4ac47516b4bcaf",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/config.json",
"sha256": "c0e62b3248e5f8847c76ea8f18e19c52db6d1cbcb4f2e636cee33063dd541043",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval, vLLM; mean of 8 seeds)",
"full": 0.822,
"quant_score": 0.733,
"retention": 0.891727493917275,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/README.md#L211",
"source_id": "quant-redhatai-qwen3-5-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 211,
"raw_line": " 73.3% | ",
"row_label_line": 209,
"row_label": "AIME 2025 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 210,
"raw_line": " 82.2% | ",
"row_label_line": 209,
"row_label": "AIME 2025 (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w4a16@7a61387:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"artifact_revision": "7a613872f394578b0b52b683ff4ac47516b4bcaf",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/config.json",
"sha256": "c0e62b3248e5f8847c76ea8f18e19c52db6d1cbcb4f2e636cee33063dd541043",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.796,
"quant_score": 0.731,
"retention": 0.9183417085427135,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/README.md#L217",
"source_id": "quant-redhatai-qwen3-5-4b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 217,
"raw_line": " 73.1% | ",
"row_label_line": 215,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/README.md",
"sha256": "5f57f7ce32c8af16ae4b5d28c9fd5e3df9c007e3fe85585287c1cec04b9d726e",
"line": 216,
"raw_line": " 79.6% | ",
"row_label_line": 215,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w4a16@7a61387:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w4a16",
"artifact_revision": "7a613872f394578b0b52b683ff4ac47516b4bcaf",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w4a16/config.json",
"sha256": "c0e62b3248e5f8847c76ea8f18e19c52db6d1cbcb4f2e636cee33063dd541043",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w4a16/blob/7a613872f394578b0b52b683ff4ac47516b4bcaf/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.945,
"quant_score": 0.942,
"retention": 0.9968253968253968,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/README.md#L192",
"source_id": "quant-redhatai-qwen3-5-4b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 192,
"raw_line": " 94.2% | ",
"row_label_line": 190,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 191,
"raw_line": " 94.5% | ",
"row_label_line": 190,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w8a8@c30ff43:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"artifact_revision": "c30ff438442b337db924cadb5136847bbf3fb95a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/config.json",
"sha256": "f8c137092e83e5461020ab7b269ddc0bdb910ecd5bb873c01a5430ac4ee19ea8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.793,
"quant_score": 0.79,
"retention": 0.9962168978562421,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/README.md#L198",
"source_id": "quant-redhatai-qwen3-5-4b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 198,
"raw_line": " 79.0% | ",
"row_label_line": 196,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 197,
"raw_line": " 79.3% | ",
"row_label_line": 196,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w8a8@c30ff43:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"artifact_revision": "c30ff438442b337db924cadb5136847bbf3fb95a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/config.json",
"sha256": "f8c137092e83e5461020ab7b269ddc0bdb910ecd5bb873c01a5430ac4ee19ea8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.883,
"quant_score": 0.877,
"retention": 0.9932049830124575,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/README.md#L204",
"source_id": "quant-redhatai-qwen3-5-4b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 204,
"raw_line": " 87.7% | ",
"row_label_line": 202,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 203,
"raw_line": " 88.3% | ",
"row_label_line": 202,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w8a8@c30ff43:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"artifact_revision": "c30ff438442b337db924cadb5136847bbf3fb95a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/config.json",
"sha256": "f8c137092e83e5461020ab7b269ddc0bdb910ecd5bb873c01a5430ac4ee19ea8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.845,
"quant_score": 0.839,
"retention": 0.9928994082840237,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/README.md#L217",
"source_id": "quant-redhatai-qwen3-5-4b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 217,
"raw_line": " 83.9% | ",
"row_label_line": 215,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 216,
"raw_line": " 84.5% | ",
"row_label_line": 215,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w8a8@c30ff43:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"artifact_revision": "c30ff438442b337db924cadb5136847bbf3fb95a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/config.json",
"sha256": "f8c137092e83e5461020ab7b269ddc0bdb910ecd5bb873c01a5430ac4ee19ea8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval, vLLM; mean of 8 seeds)",
"full": 0.822,
"quant_score": 0.812,
"retention": 0.9878345498783456,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/README.md#L223",
"source_id": "quant-redhatai-qwen3-5-4b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 223,
"raw_line": " 81.2% | ",
"row_label_line": 221,
"row_label": "AIME 2025 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 222,
"raw_line": " 82.2% | ",
"row_label_line": 221,
"row_label": "AIME 2025 (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w8a8@c30ff43:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"artifact_revision": "c30ff438442b337db924cadb5136847bbf3fb95a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/config.json",
"sha256": "f8c137092e83e5461020ab7b269ddc0bdb910ecd5bb873c01a5430ac4ee19ea8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.796,
"quant_score": 0.785,
"retention": 0.9861809045226131,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-4B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/README.md#L229",
"source_id": "quant-redhatai-qwen3-5-4b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 229,
"raw_line": " 78.5% | ",
"row_label_line": 227,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/README.md",
"sha256": "efb31a6b29d9fe1b5a80265ac667e68f24c1026e3f34197871f6d2ff341694ef",
"line": 228,
"raw_line": " 79.6% | ",
"row_label_line": 227,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-4B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-4B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-4B\" and quantized column \"RedHatAI/Qwen3.5-4B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-4B-quantized.w8a8@c30ff43:card table",
"artifact": "RedHatAI/Qwen3.5-4B-quantized.w8a8",
"artifact_revision": "c30ff438442b337db924cadb5136847bbf3fb95a",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-4B-quantized.w8a8/config.json",
"sha256": "f8c137092e83e5461020ab7b269ddc0bdb910ecd5bb873c01a5430ac4ee19ea8",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-4B-quantized.w8a8/blob/c30ff438442b337db924cadb5136847bbf3fb95a/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.947,
"quant_score": 0.945,
"retention": 0.9978880675818373,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/README.md#L161",
"source_id": "quant-redhatai-qwen3-5-9b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 161,
"raw_line": " 94.5% | ",
"row_label_line": 159,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 160,
"raw_line": " 94.7% | ",
"row_label_line": 159,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-FP8-dynamic@790f057:card table",
"artifact": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"artifact_revision": "790f0576d2d77dd5322aa0603a470bd9e3a3d1f6",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/config.json",
"sha256": "d0e02c2d648f75aa6bd7affb2e9db415ce2ca0c64b2ae8c2526f4df33c334e19",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.825,
"quant_score": 0.824,
"retention": 0.9987878787878788,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/README.md#L167",
"source_id": "quant-redhatai-qwen3-5-9b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 167,
"raw_line": " 82.4% | ",
"row_label_line": 165,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 166,
"raw_line": " 82.5% | ",
"row_label_line": 165,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-FP8-dynamic@790f057:card table",
"artifact": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"artifact_revision": "790f0576d2d77dd5322aa0603a470bd9e3a3d1f6",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/config.json",
"sha256": "d0e02c2d648f75aa6bd7affb2e9db415ce2ca0c64b2ae8c2526f4df33c334e19",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.903,
"quant_score": 0.889,
"retention": 0.9844961240310077,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/README.md#L173",
"source_id": "quant-redhatai-qwen3-5-9b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 173,
"raw_line": " 88.9% | ",
"row_label_line": 171,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 172,
"raw_line": " 90.3% | ",
"row_label_line": 171,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-FP8-dynamic@790f057:card table",
"artifact": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"artifact_revision": "790f0576d2d77dd5322aa0603a470bd9e3a3d1f6",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/config.json",
"sha256": "d0e02c2d648f75aa6bd7affb2e9db415ce2ca0c64b2ae8c2526f4df33c334e19",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.85,
"quant_score": 0.847,
"retention": 0.9964705882352941,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/README.md#L186",
"source_id": "quant-redhatai-qwen3-5-9b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 186,
"raw_line": " 84.7% | ",
"row_label_line": 184,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 185,
"raw_line": " 85.0% | ",
"row_label_line": 184,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-FP8-dynamic@790f057:card table",
"artifact": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"artifact_revision": "790f0576d2d77dd5322aa0603a470bd9e3a3d1f6",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/config.json",
"sha256": "d0e02c2d648f75aa6bd7affb2e9db415ce2ca0c64b2ae8c2526f4df33c334e19",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval, vLLM; mean of 8 seeds)",
"full": 0.883,
"quant_score": 0.879,
"retention": 0.9954699886749717,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/README.md#L192",
"source_id": "quant-redhatai-qwen3-5-9b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 192,
"raw_line": " 87.9% | ",
"row_label_line": 190,
"row_label": "AIME 2025 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 191,
"raw_line": " 88.3% | ",
"row_label_line": 190,
"row_label": "AIME 2025 (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-FP8-dynamic@790f057:card table",
"artifact": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"artifact_revision": "790f0576d2d77dd5322aa0603a470bd9e3a3d1f6",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/config.json",
"sha256": "d0e02c2d648f75aa6bd7affb2e9db415ce2ca0c64b2ae8c2526f4df33c334e19",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.84,
"quant_score": 0.838,
"retention": 0.9976190476190476,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/README.md#L198",
"source_id": "quant-redhatai-qwen3-5-9b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 198,
"raw_line": " 83.8% | ",
"row_label_line": 196,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/README.md",
"sha256": "147e60f0018ca1279b7c3f5c768acc0a55ed87c2ef9ca0741b3c285df8536736",
"line": 197,
"raw_line": " 84.0% | ",
"row_label_line": 196,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-FP8-dynamic@790f057:card table",
"artifact": "RedHatAI/Qwen3.5-9B-FP8-dynamic",
"artifact_revision": "790f0576d2d77dd5322aa0603a470bd9e3a3d1f6",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-FP8-dynamic/config.json",
"sha256": "d0e02c2d648f75aa6bd7affb2e9db415ce2ca0c64b2ae8c2526f4df33c334e19",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-FP8-dynamic/blob/790f0576d2d77dd5322aa0603a470bd9e3a3d1f6/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.944,
"quant_score": 0.946,
"retention": 1.0021186440677967,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/README.md#L192",
"source_id": "quant-redhatai-qwen3-5-9b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 192,
"raw_line": " 94.6% | ",
"row_label_line": 190,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 191,
"raw_line": " 94.4% | ",
"row_label_line": 190,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w4a16@a398088:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"artifact_revision": "a398088c4228b0ae0c8c78df88fd1e4bf445f068",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/config.json",
"sha256": "2256824b12eddbcb2703f4f571dce71753e1bfc906c6ee2a86edf618c74cddc3",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.824,
"quant_score": 0.817,
"retention": 0.991504854368932,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/README.md#L198",
"source_id": "quant-redhatai-qwen3-5-9b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 198,
"raw_line": " 81.7% | ",
"row_label_line": 196,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 197,
"raw_line": " 82.4% | ",
"row_label_line": 196,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w4a16@a398088:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"artifact_revision": "a398088c4228b0ae0c8c78df88fd1e4bf445f068",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/config.json",
"sha256": "2256824b12eddbcb2703f4f571dce71753e1bfc906c6ee2a86edf618c74cddc3",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.895,
"quant_score": 0.877,
"retention": 0.9798882681564246,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/README.md#L204",
"source_id": "quant-redhatai-qwen3-5-9b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 204,
"raw_line": " 87.7% | ",
"row_label_line": 202,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 203,
"raw_line": " 89.5% | ",
"row_label_line": 202,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w4a16@a398088:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"artifact_revision": "a398088c4228b0ae0c8c78df88fd1e4bf445f068",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/config.json",
"sha256": "2256824b12eddbcb2703f4f571dce71753e1bfc906c6ee2a86edf618c74cddc3",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.852,
"quant_score": 0.838,
"retention": 0.9835680751173709,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/README.md#L217",
"source_id": "quant-redhatai-qwen3-5-9b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 217,
"raw_line": " 83.8% | ",
"row_label_line": 215,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 216,
"raw_line": " 85.2% | ",
"row_label_line": 215,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w4a16@a398088:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"artifact_revision": "a398088c4228b0ae0c8c78df88fd1e4bf445f068",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/config.json",
"sha256": "2256824b12eddbcb2703f4f571dce71753e1bfc906c6ee2a86edf618c74cddc3",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval, vLLM; mean of 8 seeds)",
"full": 0.854,
"quant_score": 0.688,
"retention": 0.8056206088992974,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/README.md#L223",
"source_id": "quant-redhatai-qwen3-5-9b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 223,
"raw_line": " 68.8% | ",
"row_label_line": 221,
"row_label": "AIME 2025 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 222,
"raw_line": " 85.4% | ",
"row_label_line": 221,
"row_label": "AIME 2025 (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w4a16@a398088:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"artifact_revision": "a398088c4228b0ae0c8c78df88fd1e4bf445f068",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/config.json",
"sha256": "2256824b12eddbcb2703f4f571dce71753e1bfc906c6ee2a86edf618c74cddc3",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT4 W4A16 (GPTQ, symmetric, group 128; llm-compressor, compressed-tensors); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.822,
"quant_score": 0.779,
"retention": 0.9476885644768858,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/README.md#L229",
"source_id": "quant-redhatai-qwen3-5-9b-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 229,
"raw_line": " 77.9% | ",
"row_label_line": 227,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/README.md",
"sha256": "a90e13c1b57dcf02e85f05b513be775468f37adbc1163119708182f0c7650308",
"line": 228,
"raw_line": " 82.2% | ",
"row_label_line": 227,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w4a16 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w4a16\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w4a16@a398088:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w4a16",
"artifact_revision": "a398088c4228b0ae0c8c78df88fd1e4bf445f068",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w4a16/config.json",
"sha256": "2256824b12eddbcb2703f4f571dce71753e1bfc906c6ee2a86edf618c74cddc3",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w4a16/blob/a398088c4228b0ae0c8c78df88fd1e4bf445f068/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.944,
"quant_score": 0.943,
"retention": 0.9989406779661016,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/README.md#L192",
"source_id": "quant-redhatai-qwen3-5-9b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 192,
"raw_line": " 94.3% | ",
"row_label_line": 190,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 191,
"raw_line": " 94.4% | ",
"row_label_line": 190,
"row_label": "GSM8k-Platinum (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w8a8@4257ccf:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"artifact_revision": "4257ccf0e5c6f87246fd96ea2e4fb16817101558",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/config.json",
"sha256": "72ac0bc0a46143c3c779c3d224cab8de150fbde943346c29cbdadfd6c891b62d",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.824,
"quant_score": 0.82,
"retention": 0.9951456310679612,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/README.md#L198",
"source_id": "quant-redhatai-qwen3-5-9b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 198,
"raw_line": " 82.0% | ",
"row_label_line": 196,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 197,
"raw_line": " 82.4% | ",
"row_label_line": 196,
"row_label": "MMLU-Pro (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w8a8@4257ccf:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"artifact_revision": "4257ccf0e5c6f87246fd96ea2e4fb16817101558",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/config.json",
"sha256": "72ac0bc0a46143c3c779c3d224cab8de150fbde943346c29cbdadfd6c891b62d",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.895,
"quant_score": 0.896,
"retention": 1.0011173184357542,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/README.md#L204",
"source_id": "quant-redhatai-qwen3-5-9b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 204,
"raw_line": " 89.6% | ",
"row_label_line": 202,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 203,
"raw_line": " 89.5% | ",
"row_label_line": 202,
"row_label": "IFEval — prompt strict (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w8a8@4257ccf:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"artifact_revision": "4257ccf0e5c6f87246fd96ea2e4fb16817101558",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/config.json",
"sha256": "72ac0bc0a46143c3c779c3d224cab8de150fbde943346c29cbdadfd6c891b62d",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.852,
"quant_score": 0.853,
"retention": 1.0011737089201878,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/README.md#L217",
"source_id": "quant-redhatai-qwen3-5-9b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 217,
"raw_line": " 85.3% | ",
"row_label_line": 215,
"row_label": "Math 500 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 216,
"raw_line": " 85.2% | ",
"row_label_line": 215,
"row_label": "Math 500 (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w8a8@4257ccf:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"artifact_revision": "4257ccf0e5c6f87246fd96ea2e4fb16817101558",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/config.json",
"sha256": "72ac0bc0a46143c3c779c3d224cab8de150fbde943346c29cbdadfd6c891b62d",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval, vLLM; mean of 8 seeds)",
"full": 0.854,
"quant_score": 0.854,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/README.md#L223",
"source_id": "quant-redhatai-qwen3-5-9b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 223,
"raw_line": " 85.4% | ",
"row_label_line": 221,
"row_label": "AIME 2025 (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 222,
"raw_line": " 85.4% | ",
"row_label_line": 221,
"row_label": "AIME 2025 (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w8a8@4257ccf:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"artifact_revision": "4257ccf0e5c6f87246fd96ea2e4fb16817101558",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/config.json",
"sha256": "72ac0bc0a46143c3c779c3d224cab8de150fbde943346c29cbdadfd6c891b62d",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.5-9B",
"model_id": "qwen3-5-9b",
"params_b": 9.65,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; llm-compressor, compressed-tensors) (GPTQModifier W8A8); lm_head, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.822,
"quant_score": 0.823,
"retention": 1.0012165450121655,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.5-9B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/README.md#L229",
"source_id": "quant-redhatai-qwen3-5-9b-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 229,
"raw_line": " 82.3% | ",
"row_label_line": 227,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/README.md",
"sha256": "813a01f7c537542b22d18d921c9be5093d912bf79be7b02841ffe4ba6a26644c",
"line": 228,
"raw_line": " 82.2% | ",
"row_label_line": 227,
"row_label": "GPQA Diamond (0-shot)",
"column_header": "Qwen/Qwen3.5-9B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.5-9B-quantized.w8a8 model card; baseline column \"Qwen/Qwen3.5-9B\" and quantized column \"RedHatAI/Qwen3.5-9B-quantized.w8a8\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only (thinking-mode default; no explicit override stated); GSM8K-Platinum, MMLU-Pro, IFEval, MATH-500 and GPQA Diamond averaged over 3 seeds, AIME 2025 over 8 seeds (per card). IFEval instruction-level row not recorded separately (same benchmark, correlated). Each card has its own BF16 baseline run. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.5-9B-quantized.w8a8@4257ccf:card table",
"artifact": "RedHatAI/Qwen3.5-9B-quantized.w8a8",
"artifact_revision": "4257ccf0e5c6f87246fd96ea2e4fb16817101558",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.5-9B-quantized.w8a8/config.json",
"sha256": "72ac0bc0a46143c3c779c3d224cab8de150fbde943346c29cbdadfd6c891b62d",
"url": "https://huggingface.co/RedHatAI/Qwen3.5-9B-quantized.w8a8/blob/4257ccf0e5c6f87246fd96ea2e4fb16817101558/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 block-wise (E4M3, 128x128 weight blocks, dynamic activations; config.json quant_method fp8; embeddings, lm_head, norms, MoE router/shared-expert gates, parts of linear attention, vision encoder and MTP excluded per modules_to_not_convert)",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness gsm8k_platinum_cot_llama, chat completions; mean of 3 seeds)",
"full": 0.9573,
"quant_score": 0.9597,
"retention": 1.0025070510811658,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/README.md#L1066",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1066,
"raw_line": "| GSM8k Platinum | 95.73 | 95.97 | 100.26 |",
"row_label": "GSM8k Platinum",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1066,
"raw_line": "| GSM8k Platinum | 95.73 | 95.97 | 100.26 |",
"row_label": "GSM8k Platinum",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8\" are from the same card table. Red Hat accuracy table appended to a card that otherwise reproduces Qwen's own model card text. This card does not restate the protocol; its BF16 baseline column is identical to the baseline in Red Hat's Qwen3.6-35B-A3B-FP8-dynamic and -NVFP4 cards, which document it: Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8@ea24706:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8",
"artifact_revision": "ea247061a1c961ddc12daff63a326b345a0489b4",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/config.json",
"sha256": "570ef7ea45a7e1d3de2b1d3c70c4ac3562d0e768acdc195778cb4f4d95025845",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 block-wise (E4M3, 128x128 weight blocks, dynamic activations; config.json quant_method fp8; embeddings, lm_head, norms, MoE router/shared-expert gates, parts of linear attention, vision encoder and MTP excluded per modules_to_not_convert)",
"metric": "IFEval 0-shot (lm-evaluation-harness ifeval, chat completions; reported metric variant not stated; mean of 3 seeds)",
"full": 0.9309,
"quant_score": 0.9273,
"retention": 0.9961327747341283,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/README.md#L1067",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1067,
"raw_line": "| IfEval | 93.09 | 92.73 | 99.61 |",
"row_label": "IfEval",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1067,
"raw_line": "| IfEval | 93.09 | 92.73 | 99.61 |",
"row_label": "IfEval",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8\" are from the same card table. Red Hat accuracy table appended to a card that otherwise reproduces Qwen's own model card text. This card does not restate the protocol; its BF16 baseline column is identical to the baseline in Red Hat's Qwen3.6-35B-A3B-FP8-dynamic and -NVFP4 cards, which document it: Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8@ea24706:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8",
"artifact_revision": "ea247061a1c961ddc12daff63a326b345a0489b4",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/config.json",
"sha256": "570ef7ea45a7e1d3de2b1d3c70c4ac3562d0e768acdc195778cb4f4d95025845",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 block-wise (E4M3, 128x128 weight blocks, dynamic activations; config.json quant_method fp8; embeddings, lm_head, norms, MoE router/shared-expert gates, parts of linear attention, vision encoder and MTP excluded per modules_to_not_convert)",
"metric": "AIME 2025 0-shot (lighteval via litellm; mean of 8 seeds)",
"full": 0.9292,
"quant_score": 0.9292,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/README.md#L1068",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1068,
"raw_line": "| AIME 2025 | 92.92 | 92.92 | 100.00 |",
"row_label": "AIME 2025",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1068,
"raw_line": "| AIME 2025 | 92.92 | 92.92 | 100.00 |",
"row_label": "AIME 2025",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8\" are from the same card table. Red Hat accuracy table appended to a card that otherwise reproduces Qwen's own model card text. This card does not restate the protocol; its BF16 baseline column is identical to the baseline in Red Hat's Qwen3.6-35B-A3B-FP8-dynamic and -NVFP4 cards, which document it: Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8@ea24706:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8",
"artifact_revision": "ea247061a1c961ddc12daff63a326b345a0489b4",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/config.json",
"sha256": "570ef7ea45a7e1d3de2b1d3c70c4ac3562d0e768acdc195778cb4f4d95025845",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 block-wise (E4M3, 128x128 weight blocks, dynamic activations; config.json quant_method fp8; embeddings, lm_head, norms, MoE router/shared-expert gates, parts of linear attention, vision encoder and MTP excluded per modules_to_not_convert)",
"metric": "GPQA Diamond 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8451,
"quant_score": 0.8586,
"retention": 1.0159744408945688,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/README.md#L1069",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1069,
"raw_line": "| GPQA diamond | 84.51 | 85.86 | 101.59 |",
"row_label": "GPQA diamond",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1069,
"raw_line": "| GPQA diamond | 84.51 | 85.86 | 101.59 |",
"row_label": "GPQA diamond",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8\" are from the same card table. Red Hat accuracy table appended to a card that otherwise reproduces Qwen's own model card text. This card does not restate the protocol; its BF16 baseline column is identical to the baseline in Red Hat's Qwen3.6-35B-A3B-FP8-dynamic and -NVFP4 cards, which document it: Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8@ea24706:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8",
"artifact_revision": "ea247061a1c961ddc12daff63a326b345a0489b4",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/config.json",
"sha256": "570ef7ea45a7e1d3de2b1d3c70c4ac3562d0e768acdc195778cb4f4d95025845",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 block-wise (E4M3, 128x128 weight blocks, dynamic activations; config.json quant_method fp8; embeddings, lm_head, norms, MoE router/shared-expert gates, parts of linear attention, vision encoder and MTP excluded per modules_to_not_convert)",
"metric": "MATH-500 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.848,
"quant_score": 0.848,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/README.md#L1070",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1070,
"raw_line": "| Math 500 | 84.80 | 84.80 | 100.00 |",
"row_label": "Math 500",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1070,
"raw_line": "| Math 500 | 84.80 | 84.80 | 100.00 |",
"row_label": "Math 500",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8\" are from the same card table. Red Hat accuracy table appended to a card that otherwise reproduces Qwen's own model card text. This card does not restate the protocol; its BF16 baseline column is identical to the baseline in Red Hat's Qwen3.6-35B-A3B-FP8-dynamic and -NVFP4 cards, which document it: Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8@ea24706:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8",
"artifact_revision": "ea247061a1c961ddc12daff63a326b345a0489b4",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/config.json",
"sha256": "570ef7ea45a7e1d3de2b1d3c70c4ac3562d0e768acdc195778cb4f4d95025845",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 block-wise (E4M3, 128x128 weight blocks, dynamic activations; config.json quant_method fp8; embeddings, lm_head, norms, MoE router/shared-expert gates, parts of linear attention, vision encoder and MTP excluded per modules_to_not_convert)",
"metric": "LiveCodeBench v6 code generation 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.7733,
"quant_score": 0.7657,
"retention": 0.9901719901719902,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/README.md#L1071",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1071,
"raw_line": "| Lcb Codegeneration V6 | 77.33 | 76.57 | 99.01 |",
"row_label": "Lcb Codegeneration V6",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1071,
"raw_line": "| Lcb Codegeneration V6 | 77.33 | 76.57 | 99.01 |",
"row_label": "Lcb Codegeneration V6",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8\" are from the same card table. Red Hat accuracy table appended to a card that otherwise reproduces Qwen's own model card text. This card does not restate the protocol; its BF16 baseline column is identical to the baseline in Red Hat's Qwen3.6-35B-A3B-FP8-dynamic and -NVFP4 cards, which document it: Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8@ea24706:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8",
"artifact_revision": "ea247061a1c961ddc12daff63a326b345a0489b4",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/config.json",
"sha256": "570ef7ea45a7e1d3de2b1d3c70c4ac3562d0e768acdc195778cb4f4d95025845",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 block-wise (E4M3, 128x128 weight blocks, dynamic activations; config.json quant_method fp8; embeddings, lm_head, norms, MoE router/shared-expert gates, parts of linear attention, vision encoder and MTP excluded per modules_to_not_convert)",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness mmlu_pro_chat, chat completions; mean of 3 seeds)",
"full": 0.8532,
"quant_score": 0.8528,
"retention": 0.9995311767463667,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/README.md#L1072",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1072,
"raw_line": "| MMLU Pro Chat | 85.32 | 85.28 | 99.95 |",
"row_label": "MMLU Pro Chat",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/README.md",
"sha256": "01b30b134f14b4f855ff7d4146d13bd880409f705cf74d4e1fdb8eaee63bdb6d",
"line": 1072,
"raw_line": "| MMLU Pro Chat | 85.32 | 85.28 | 99.95 |",
"row_label": "MMLU Pro Chat",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8\" are from the same card table. Red Hat accuracy table appended to a card that otherwise reproduces Qwen's own model card text. This card does not restate the protocol; its BF16 baseline column is identical to the baseline in Red Hat's Qwen3.6-35B-A3B-FP8-dynamic and -NVFP4 cards, which document it: Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8@ea24706:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8",
"artifact_revision": "ea247061a1c961ddc12daff63a326b345a0489b4",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8/config.json",
"sha256": "570ef7ea45a7e1d3de2b1d3c70c4ac3562d0e768acdc195778cb4f4d95025845",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8/blob/ea247061a1c961ddc12daff63a326b345a0489b4/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness gsm8k_platinum_cot_llama, chat completions; mean of 3 seeds)",
"full": 0.9573,
"quant_score": 0.9562,
"retention": 0.9988509349211324,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/README.md#L181",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 181,
"raw_line": "| GSM8k Platinum | 95.73 | 95.62 | 99.88 |",
"row_label": "GSM8k Platinum",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 181,
"raw_line": "| GSM8k Platinum | 95.73 | 95.62 | 99.88 |",
"row_label": "GSM8k Platinum",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic@6496d54:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic",
"artifact_revision": "6496d54f9127add4e6a62b7c5e3c5579ddb04ac7",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/config.json",
"sha256": "22c7c95aaac45b16b6ba77685227f6cf3cf54807bca998092840e00a12f68d94",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "IFEval 0-shot (lm-evaluation-harness ifeval, chat completions; reported metric variant not stated; mean of 3 seeds)",
"full": 0.9309,
"quant_score": 0.9269,
"retention": 0.9957030830379203,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/README.md#L182",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 182,
"raw_line": "| IfEval | 93.09 | 92.69 | 99.57 |",
"row_label": "IfEval",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 182,
"raw_line": "| IfEval | 93.09 | 92.69 | 99.57 |",
"row_label": "IfEval",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic@6496d54:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic",
"artifact_revision": "6496d54f9127add4e6a62b7c5e3c5579ddb04ac7",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/config.json",
"sha256": "22c7c95aaac45b16b6ba77685227f6cf3cf54807bca998092840e00a12f68d94",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "AIME 2025 0-shot (lighteval via litellm; mean of 8 seeds)",
"full": 0.9292,
"quant_score": 0.925,
"retention": 0.9954799827808868,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/README.md#L183",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 183,
"raw_line": "| AIME 2025 | 92.92 | 92.50 | 99.55 |",
"row_label": "AIME 2025",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 183,
"raw_line": "| AIME 2025 | 92.92 | 92.50 | 99.55 |",
"row_label": "AIME 2025",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic@6496d54:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic",
"artifact_revision": "6496d54f9127add4e6a62b7c5e3c5579ddb04ac7",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/config.json",
"sha256": "22c7c95aaac45b16b6ba77685227f6cf3cf54807bca998092840e00a12f68d94",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8451,
"quant_score": 0.8485,
"retention": 1.0040231925215952,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/README.md#L184",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 184,
"raw_line": "| GPQA diamond | 84.51 | 84.85 | 100.40 |",
"row_label": "GPQA diamond",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 184,
"raw_line": "| GPQA diamond | 84.51 | 84.85 | 100.40 |",
"row_label": "GPQA diamond",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic@6496d54:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic",
"artifact_revision": "6496d54f9127add4e6a62b7c5e3c5579ddb04ac7",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/config.json",
"sha256": "22c7c95aaac45b16b6ba77685227f6cf3cf54807bca998092840e00a12f68d94",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MATH-500 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.848,
"quant_score": 0.8473,
"retention": 0.9991745283018869,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/README.md#L185",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 185,
"raw_line": "| Math 500 | 84.80 | 84.73 | 99.92 |",
"row_label": "Math 500",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 185,
"raw_line": "| Math 500 | 84.80 | 84.73 | 99.92 |",
"row_label": "Math 500",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic@6496d54:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic",
"artifact_revision": "6496d54f9127add4e6a62b7c5e3c5579ddb04ac7",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/config.json",
"sha256": "22c7c95aaac45b16b6ba77685227f6cf3cf54807bca998092840e00a12f68d94",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "LiveCodeBench v6 code generation 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.7733,
"quant_score": 0.7886,
"retention": 1.0197853355748092,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/README.md#L186",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 186,
"raw_line": "| Lcb Codegeneration V6 | 77.33 | 78.86 | 101.97 |",
"row_label": "Lcb Codegeneration V6",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 186,
"raw_line": "| Lcb Codegeneration V6 | 77.33 | 78.86 | 101.97 |",
"row_label": "Lcb Codegeneration V6",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic@6496d54:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic",
"artifact_revision": "6496d54f9127add4e6a62b7c5e3c5579ddb04ac7",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/config.json",
"sha256": "22c7c95aaac45b16b6ba77685227f6cf3cf54807bca998092840e00a12f68d94",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "FP8 W8A8 dynamic (E4M3; weights static per-channel, activations dynamic per-token; llm-compressor FP8_DYNAMIC, compressed-tensors); lm_head, MoE router/shared-expert gates, linear-attention (Gated DeltaNet), vision-encoder and MTP layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness mmlu_pro_chat, chat completions; mean of 3 seeds)",
"full": 0.8532,
"quant_score": 0.8536,
"retention": 1.0004688232536334,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/README.md#L187",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 187,
"raw_line": "| MMLU Pro Chat | 85.32 | 85.36 | 100.05 |",
"row_label": "MMLU Pro Chat",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/README.md",
"sha256": "de4671f9a3f20f1e3350a5e3f2575e717b8127f13112b7a8db57902ed600f909",
"line": 187,
"raw_line": "| MMLU Pro Chat | 85.32 | 85.36 | 100.05 |",
"row_label": "MMLU Pro Chat",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic@6496d54:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic",
"artifact_revision": "6496d54f9127add4e6a62b7c5e3c5579ddb04ac7",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-FP8-dynamic/config.json",
"sha256": "22c7c95aaac45b16b6ba77685227f6cf3cf54807bca998092840e00a12f68d94",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic/blob/6496d54f9127add4e6a62b7c5e3c5579ddb04ac7/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, embeddings, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision layers unquantized",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness gsm8k_platinum_cot_llama, chat completions; mean of 3 seeds)",
"full": 0.9573,
"quant_score": 0.9608,
"retention": 1.0036561161600335,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/README.md#L164",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 164,
"raw_line": "| GSM8k Platinum (0-shot) | 95.73 | 96.08 | 100.37 |",
"row_label": "GSM8k Platinum (0-shot)",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 164,
"raw_line": "| GSM8k Platinum (0-shot) | 95.73 | 96.08 | 100.37 |",
"row_label": "GSM8k Platinum (0-shot)",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-NVFP4 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-NVFP4@e94a930:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
"artifact_revision": "e94a930c8e9b8a7d3763e449289c261c9c7d3e5e",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/config.json",
"sha256": "2af7513fb092e80e64480b8a9155c1347f6a742c958aa35448b0842f5fdaeac8",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, embeddings, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision layers unquantized",
"metric": "IFEval 0-shot (lm-evaluation-harness ifeval, chat completions; reported metric variant not stated; mean of 3 seeds)",
"full": 0.9309,
"quant_score": 0.9245,
"retention": 0.9931249328606725,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/README.md#L165",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 165,
"raw_line": "| IfEval (0-shot) | 93.09 | 92.45 | 99.31 |",
"row_label": "IfEval (0-shot)",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 165,
"raw_line": "| IfEval (0-shot) | 93.09 | 92.45 | 99.31 |",
"row_label": "IfEval (0-shot)",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-NVFP4 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-NVFP4@e94a930:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
"artifact_revision": "e94a930c8e9b8a7d3763e449289c261c9c7d3e5e",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/config.json",
"sha256": "2af7513fb092e80e64480b8a9155c1347f6a742c958aa35448b0842f5fdaeac8",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, embeddings, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision layers unquantized",
"metric": "AIME 2025 0-shot (lighteval via litellm; mean of 8 seeds)",
"full": 0.9292,
"quant_score": 0.9125,
"retention": 0.982027550581145,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/README.md#L166",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 166,
"raw_line": "| AIME 2025 | 92.92 | 91.25 | 98.21 |",
"row_label": "AIME 2025",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 166,
"raw_line": "| AIME 2025 | 92.92 | 91.25 | 98.21 |",
"row_label": "AIME 2025",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-NVFP4 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-NVFP4@e94a930:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
"artifact_revision": "e94a930c8e9b8a7d3763e449289c261c9c7d3e5e",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/config.json",
"sha256": "2af7513fb092e80e64480b8a9155c1347f6a742c958aa35448b0842f5fdaeac8",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, embeddings, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision layers unquantized",
"metric": "GPQA Diamond 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.8451,
"quant_score": 0.8468,
"retention": 1.0020115962607976,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/README.md#L167",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 167,
"raw_line": "| GPQA diamond | 84.51 | 84.68 | 100.20 |",
"row_label": "GPQA diamond",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 167,
"raw_line": "| GPQA diamond | 84.51 | 84.68 | 100.20 |",
"row_label": "GPQA diamond",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-NVFP4 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-NVFP4@e94a930:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
"artifact_revision": "e94a930c8e9b8a7d3763e449289c261c9c7d3e5e",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/config.json",
"sha256": "2af7513fb092e80e64480b8a9155c1347f6a742c958aa35448b0842f5fdaeac8",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, embeddings, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision layers unquantized",
"metric": "MATH-500 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.848,
"quant_score": 0.85,
"retention": 1.0023584905660377,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/README.md#L168",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 168,
"raw_line": "| Math 500 | 84.80 | 85.00 | 100.24 |",
"row_label": "Math 500",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 168,
"raw_line": "| Math 500 | 84.80 | 85.00 | 100.24 |",
"row_label": "Math 500",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-NVFP4 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-NVFP4@e94a930:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
"artifact_revision": "e94a930c8e9b8a7d3763e449289c261c9c7d3e5e",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/config.json",
"sha256": "2af7513fb092e80e64480b8a9155c1347f6a742c958aa35448b0842f5fdaeac8",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, embeddings, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision layers unquantized",
"metric": "LiveCodeBench v6 code generation 0-shot (lighteval via litellm; mean of 3 seeds)",
"full": 0.7733,
"quant_score": 0.7467,
"retention": 0.9656019656019657,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/README.md#L169",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 169,
"raw_line": "| Lcb Codegeneration V6 | 77.33 | 74.67 | 96.55 |",
"row_label": "Lcb Codegeneration V6",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 169,
"raw_line": "| Lcb Codegeneration V6 | 77.33 | 74.67 | 96.55 |",
"row_label": "Lcb Codegeneration V6",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-NVFP4 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-NVFP4@e94a930:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
"artifact_revision": "e94a930c8e9b8a7d3763e449289c261c9c7d3e5e",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/config.json",
"sha256": "2af7513fb092e80e64480b8a9155c1347f6a742c958aa35448b0842f5fdaeac8",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.6-35B-A3B",
"model_id": "qwen3-6-35b-a3b",
"params_b": 35.5,
"quant": "NVFP4 W4A4 (FP4 E2M1 weights and activations, group 16; llm-compressor QuantizationModifier NVFP4, compressed-tensors); lm_head, embeddings, MoE router/shared-expert gates, linear-attention (Gated DeltaNet) and vision layers unquantized",
"metric": "MMLU-Pro 0-shot (lm-evaluation-harness mmlu_pro_chat, chat completions; mean of 3 seeds)",
"full": 0.8532,
"quant_score": 0.847,
"retention": 0.9927332395686826,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.6-35B-A3B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/README.md#L170",
"source_id": "quant-redhatai-qwen3-6-35b-a3b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 170,
"raw_line": "| MMLU Pro Chat | 85.32 | 84.70 | 99.28 |",
"row_label": "MMLU Pro Chat",
"column_header": "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/README.md",
"sha256": "9e4501d2f1e3d32e0e327cbd035495e5e93d0ec3167b1f5f6a064bdf9471fb14",
"line": 170,
"raw_line": "| MMLU Pro Chat | 85.32 | 84.70 | 99.28 |",
"row_label": "MMLU Pro Chat",
"column_header": "Qwen/Qwen3.6-35B-A3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.6-35B-A3B-NVFP4 model card; baseline column \"Qwen/Qwen3.6-35B-A3B\" and quantized column \"RedHatAI/Qwen3.6-35B-A3B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM server with --reasoning-parser qwen3 --language-model-only --max-model-len 96000 (thinking-mode default); sampling temperature 1.0, top_p 0.95, top_k 20, presence_penalty 1.5, max 64000 new tokens; 3 seeds (42, 1234, 4158), AIME 2025 8 seeds (per Red Hat Qwen3.6-35B-A3B FP8-dynamic/NVFP4 cards). BFCLv4 and SWE-bench Verified rows in the same table are not used (outside selected benchmarks; agentic protocol not described in this card). Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.6-35B-A3B-NVFP4@e94a930:card table",
"artifact": "RedHatAI/Qwen3.6-35B-A3B-NVFP4",
"artifact_revision": "e94a930c8e9b8a7d3763e449289c261c9c7d3e5e",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.6-35B-A3B-NVFP4/config.json",
"sha256": "2af7513fb092e80e64480b8a9155c1347f6a742c958aa35448b0842f5fdaeac8",
"url": "https://huggingface.co/RedHatAI/Qwen3.6-35B-A3B-NVFP4/blob/e94a930c8e9b8a7d3763e449289c261c9c7d3e5e/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "INT4 W4A16 (AWQ smoothing + GPTQ, symmetric, group 128; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP, linear-attention in_proj_a/b and some linear_attn modules unquantized per config; llm-compressor)",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.9224,
"quant_score": 0.9193,
"retention": 0.9966392020815265,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/README.md#L185",
"source_id": "quant-redhatai-qwen3-8-27b-int4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 185,
"raw_line": " 91.93% | ",
"row_label_line": 183,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/Qwen3.8-27B-INT4",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 184,
"raw_line": " 92.24% | ",
"row_label_line": 183,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "Qwen/Qwen3.8-27B (BF16)",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-INT4 model card; baseline column \"Qwen/Qwen3.8-27B (BF16)\" and quantized column \"RedHatAI/Qwen3.8-27B-INT4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Card: \"recovery is computed against the BF16 model\". Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-INT4@2f0fa48:card table",
"artifact": "RedHatAI/Qwen3.8-27B-INT4",
"artifact_revision": "2f0fa48fea4949b56685ddfc48c05dd81c7d772b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/config.json",
"sha256": "5498b42f9ea96d2669709c177e29122997ce14786a7530fe1ea1b340e535776c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "INT4 W4A16 (AWQ smoothing + GPTQ, symmetric, group 128; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP, linear-attention in_proj_a/b and some linear_attn modules unquantized per config; llm-compressor)",
"metric": "MMLU-Pro exact-match (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.8446,
"quant_score": 0.8345,
"retention": 0.9880416765332702,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/README.md#L191",
"source_id": "quant-redhatai-qwen3-8-27b-int4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 191,
"raw_line": " 83.45% | ",
"row_label_line": 189,
"row_label": "MMLU-Pro (exact-match)",
"column_header": "RedHatAI/Qwen3.8-27B-INT4",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 190,
"raw_line": " 84.46% | ",
"row_label_line": 189,
"row_label": "MMLU-Pro (exact-match)",
"column_header": "Qwen/Qwen3.8-27B (BF16)",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-INT4 model card; baseline column \"Qwen/Qwen3.8-27B (BF16)\" and quantized column \"RedHatAI/Qwen3.8-27B-INT4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Card: \"recovery is computed against the BF16 model\". Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-INT4@2f0fa48:card table",
"artifact": "RedHatAI/Qwen3.8-27B-INT4",
"artifact_revision": "2f0fa48fea4949b56685ddfc48c05dd81c7d772b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/config.json",
"sha256": "5498b42f9ea96d2669709c177e29122997ce14786a7530fe1ea1b340e535776c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "INT4 W4A16 (AWQ smoothing + GPTQ, symmetric, group 128; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP, linear-attention in_proj_a/b and some linear_attn modules unquantized per config; llm-compressor)",
"metric": "GSM8K-Platinum strict-match (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.9575,
"quant_score": 0.9677,
"retention": 1.0106527415143602,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/README.md#L198",
"source_id": "quant-redhatai-qwen3-8-27b-int4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 198,
"raw_line": " 96.77% | ",
"row_label_line": 196,
"row_label": "GSM8K Platinum (strict-match)",
"column_header": "RedHatAI/Qwen3.8-27B-INT4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 197,
"raw_line": " 95.75% | ",
"row_label_line": 196,
"row_label": "GSM8K Platinum (strict-match)",
"column_header": "Qwen/Qwen3.8-27B (BF16)",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-INT4 model card; baseline column \"Qwen/Qwen3.8-27B (BF16)\" and quantized column \"RedHatAI/Qwen3.8-27B-INT4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Card: \"recovery is computed against the BF16 model\". Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-INT4@2f0fa48:card table",
"artifact": "RedHatAI/Qwen3.8-27B-INT4",
"artifact_revision": "2f0fa48fea4949b56685ddfc48c05dd81c7d772b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/config.json",
"sha256": "5498b42f9ea96d2669709c177e29122997ce14786a7530fe1ea1b340e535776c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "INT4 W4A16 (AWQ smoothing + GPTQ, symmetric, group 128; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP, linear-attention in_proj_a/b and some linear_attn modules unquantized per config; llm-compressor)",
"metric": "MATH-500 pass@1 (lighteval, vLLM; mean of 3 seeds)",
"full": 0.8373,
"quant_score": 0.8333,
"retention": 0.9952227397587483,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/README.md#L204",
"source_id": "quant-redhatai-qwen3-8-27b-int4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 204,
"raw_line": " 83.33% | ",
"row_label_line": 202,
"row_label": "MATH-500 (pass@1)",
"column_header": "RedHatAI/Qwen3.8-27B-INT4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 203,
"raw_line": " 83.73% | ",
"row_label_line": 202,
"row_label": "MATH-500 (pass@1)",
"column_header": "Qwen/Qwen3.8-27B (BF16)",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-INT4 model card; baseline column \"Qwen/Qwen3.8-27B (BF16)\" and quantized column \"RedHatAI/Qwen3.8-27B-INT4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Card: \"recovery is computed against the BF16 model\". Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-INT4@2f0fa48:card table",
"artifact": "RedHatAI/Qwen3.8-27B-INT4",
"artifact_revision": "2f0fa48fea4949b56685ddfc48c05dd81c7d772b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/config.json",
"sha256": "5498b42f9ea96d2669709c177e29122997ce14786a7530fe1ea1b340e535776c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "INT4 W4A16 (AWQ smoothing + GPTQ, symmetric, group 128; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP, linear-attention in_proj_a/b and some linear_attn modules unquantized per config; llm-compressor)",
"metric": "GPQA Diamond pass@1 (lighteval, vLLM; mean of 3 seeds)",
"full": 0.8923,
"quant_score": 0.8788,
"retention": 0.9848705592289589,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/README.md#L210",
"source_id": "quant-redhatai-qwen3-8-27b-int4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 210,
"raw_line": " 87.88% | ",
"row_label_line": 208,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "RedHatAI/Qwen3.8-27B-INT4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 209,
"raw_line": " 89.23% | ",
"row_label_line": 208,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "Qwen/Qwen3.8-27B (BF16)",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-INT4 model card; baseline column \"Qwen/Qwen3.8-27B (BF16)\" and quantized column \"RedHatAI/Qwen3.8-27B-INT4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Card: \"recovery is computed against the BF16 model\". Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-INT4@2f0fa48:card table",
"artifact": "RedHatAI/Qwen3.8-27B-INT4",
"artifact_revision": "2f0fa48fea4949b56685ddfc48c05dd81c7d772b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/config.json",
"sha256": "5498b42f9ea96d2669709c177e29122997ce14786a7530fe1ea1b340e535776c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "INT4 W4A16 (AWQ smoothing + GPTQ, symmetric, group 128; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP, linear-attention in_proj_a/b and some linear_attn modules unquantized per config; llm-compressor)",
"metric": "AIME 2025 pass@1 (lighteval, vLLM; mean of 8 seeds)",
"full": 0.9542,
"quant_score": 0.9417,
"retention": 0.9869000209599664,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/README.md#L216",
"source_id": "quant-redhatai-qwen3-8-27b-int4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 216,
"raw_line": " 94.17% | ",
"row_label_line": 214,
"row_label": "AIME 2025 (pass@1)",
"column_header": "RedHatAI/Qwen3.8-27B-INT4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/README.md",
"sha256": "047b4e8eed70e6cddfa8e5576a9c2b5df8496885abb5362a0e3d701bb44f6f9d",
"line": 215,
"raw_line": " 95.42% | ",
"row_label_line": 214,
"row_label": "AIME 2025 (pass@1)",
"column_header": "Qwen/Qwen3.8-27B (BF16)",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-INT4 model card; baseline column \"Qwen/Qwen3.8-27B (BF16)\" and quantized column \"RedHatAI/Qwen3.8-27B-INT4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Card: \"recovery is computed against the BF16 model\". Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-INT4@2f0fa48:card table",
"artifact": "RedHatAI/Qwen3.8-27B-INT4",
"artifact_revision": "2f0fa48fea4949b56685ddfc48c05dd81c7d772b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-INT4/config.json",
"sha256": "5498b42f9ea96d2669709c177e29122997ce14786a7530fe1ea1b340e535776c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4/blob/2f0fa48fea4949b56685ddfc48c05dd81c7d772b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "MXFP4 W4A4 (FP4 weights and activations, group 32; AWQ smoothing + GPTQ; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "IFEval prompt-level strict 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.9119,
"quant_score": 0.9088,
"retention": 0.9966005044412765,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/README.md#L137",
"source_id": "quant-redhatai-qwen3-8-27b-mxfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 137,
"raw_line": " 90.88% | ",
"row_label_line": 135,
"row_label": "IFEval (prompt-level strict)",
"column_header": "RedHatAI/Qwen3.8-27B-MXFP4",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 136,
"raw_line": " 91.19% | ",
"row_label_line": 135,
"row_label": "IFEval (prompt-level strict)",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-MXFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-MXFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x NVIDIA B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-MXFP4@651eed9:card table",
"artifact": "RedHatAI/Qwen3.8-27B-MXFP4",
"artifact_revision": "651eed99386dae5645cd4949024e8ebd163a533b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/config.json",
"sha256": "18f512890fa5882d302cfc5b41beeffd0b249bde718ffb4ee35129a7e59ba96c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "MXFP4 W4A4 (FP4 weights and activations, group 32; AWQ smoothing + GPTQ; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "GSM8K-Platinum strict-match 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.9625,
"quant_score": 0.9518,
"retention": 0.9888831168831168,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/README.md#L144",
"source_id": "quant-redhatai-qwen3-8-27b-mxfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 144,
"raw_line": " 95.18% | ",
"row_label_line": 142,
"row_label": "GSM8K Platinum (strict-match)",
"column_header": "RedHatAI/Qwen3.8-27B-MXFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 143,
"raw_line": " 96.25% | ",
"row_label_line": 142,
"row_label": "GSM8K Platinum (strict-match)",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-MXFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-MXFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x NVIDIA B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-MXFP4@651eed9:card table",
"artifact": "RedHatAI/Qwen3.8-27B-MXFP4",
"artifact_revision": "651eed99386dae5645cd4949024e8ebd163a533b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/config.json",
"sha256": "18f512890fa5882d302cfc5b41beeffd0b249bde718ffb4ee35129a7e59ba96c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "MXFP4 W4A4 (FP4 weights and activations, group 32; AWQ smoothing + GPTQ; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "MATH-500 pass@1 (lighteval, vLLM; mean of 3 seeds)",
"full": 0.8367,
"quant_score": 0.8467,
"retention": 1.0119517150711128,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/README.md#L150",
"source_id": "quant-redhatai-qwen3-8-27b-mxfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 150,
"raw_line": " 84.67% | ",
"row_label_line": 148,
"row_label": "MATH-500 (pass@1)",
"column_header": "RedHatAI/Qwen3.8-27B-MXFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 149,
"raw_line": " 83.67% | ",
"row_label_line": 148,
"row_label": "MATH-500 (pass@1)",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-MXFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-MXFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x NVIDIA B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-MXFP4@651eed9:card table",
"artifact": "RedHatAI/Qwen3.8-27B-MXFP4",
"artifact_revision": "651eed99386dae5645cd4949024e8ebd163a533b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/config.json",
"sha256": "18f512890fa5882d302cfc5b41beeffd0b249bde718ffb4ee35129a7e59ba96c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "MXFP4 W4A4 (FP4 weights and activations, group 32; AWQ smoothing + GPTQ; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "AIME 2025 pass@1 (lighteval, vLLM; mean of 8 seeds)",
"full": 0.9667,
"quant_score": 0.925,
"retention": 0.9568635564290887,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/README.md#L156",
"source_id": "quant-redhatai-qwen3-8-27b-mxfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 156,
"raw_line": " 92.50% | ",
"row_label_line": 154,
"row_label": "AIME 2025 (pass@1)",
"column_header": "RedHatAI/Qwen3.8-27B-MXFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 155,
"raw_line": " 96.67% | ",
"row_label_line": 154,
"row_label": "AIME 2025 (pass@1)",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-MXFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-MXFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x NVIDIA B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-MXFP4@651eed9:card table",
"artifact": "RedHatAI/Qwen3.8-27B-MXFP4",
"artifact_revision": "651eed99386dae5645cd4949024e8ebd163a533b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/config.json",
"sha256": "18f512890fa5882d302cfc5b41beeffd0b249bde718ffb4ee35129a7e59ba96c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "MXFP4 W4A4 (FP4 weights and activations, group 32; AWQ smoothing + GPTQ; FP8 static per-tensor KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "GPQA Diamond pass@1 (lighteval, vLLM; mean of 3 seeds)",
"full": 0.8956,
"quant_score": 0.8771,
"retention": 0.979343456900402,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/README.md#L162",
"source_id": "quant-redhatai-qwen3-8-27b-mxfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 162,
"raw_line": " 87.71% | ",
"row_label_line": 160,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "RedHatAI/Qwen3.8-27B-MXFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/README.md",
"sha256": "2cdb58cce6d92ddd5acd02cf157f9d4dc8b3cd1b388ed84d4e07168eef867927",
"line": 161,
"raw_line": " 89.56% | ",
"row_label_line": 160,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-MXFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-MXFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x NVIDIA B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-MXFP4@651eed9:card table",
"artifact": "RedHatAI/Qwen3.8-27B-MXFP4",
"artifact_revision": "651eed99386dae5645cd4949024e8ebd163a533b",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-MXFP4/config.json",
"sha256": "18f512890fa5882d302cfc5b41beeffd0b249bde718ffb4ee35129a7e59ba96c",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-MXFP4/blob/651eed99386dae5645cd4949024e8ebd163a533b/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "Mixed NVFP4/FP8 (MLP projections NVFP4 group 16; attention/linear-attention projections and last 8 MLP layers FP8 dynamic; AWQ smoothing + GPTQ; FP8 KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "GSM8K-Platinum 0-shot (lm-evaluation-harness, vLLM; mean of 3 seeds)",
"full": 0.9625,
"quant_score": 0.9672,
"retention": 1.0048831168831167,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/README.md#L161",
"source_id": "quant-redhatai-qwen3-8-27b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 161,
"raw_line": " 96.72% | ",
"row_label_line": 159,
"row_label": "GSM8K Platinum",
"column_header": "RedHatAI/Qwen3.8-27B-NVFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 160,
"raw_line": " 96.25% | ",
"row_label_line": 159,
"row_label": "GSM8K Platinum",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-NVFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. SWE Bench row (Inspect AI; variant not stated) and the separate \"NVFP4 build comparison\" table (unsloth/Inferact builds, no BF16 column) are not used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-NVFP4@d23c6ff:card table",
"artifact": "RedHatAI/Qwen3.8-27B-NVFP4",
"artifact_revision": "d23c6ff198a005532746610e9d719c9f6d27a2b1",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/config.json",
"sha256": "f88aad53f300b6445ca79249c9c98ec45f82576bfec27f7354ba1729d6788776",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "Mixed NVFP4/FP8 (MLP projections NVFP4 group 16; attention/linear-attention projections and last 8 MLP layers FP8 dynamic; AWQ smoothing + GPTQ; FP8 KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "MATH-500 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.8367,
"quant_score": 0.8427,
"retention": 1.0071710290426676,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/README.md#L167",
"source_id": "quant-redhatai-qwen3-8-27b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 167,
"raw_line": " 84.27% | ",
"row_label_line": 165,
"row_label": "MATH-500",
"column_header": "RedHatAI/Qwen3.8-27B-NVFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 166,
"raw_line": " 83.67% | ",
"row_label_line": 165,
"row_label": "MATH-500",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-NVFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. SWE Bench row (Inspect AI; variant not stated) and the separate \"NVFP4 build comparison\" table (unsloth/Inferact builds, no BF16 column) are not used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-NVFP4@d23c6ff:card table",
"artifact": "RedHatAI/Qwen3.8-27B-NVFP4",
"artifact_revision": "d23c6ff198a005532746610e9d719c9f6d27a2b1",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/config.json",
"sha256": "f88aad53f300b6445ca79249c9c98ec45f82576bfec27f7354ba1729d6788776",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "Mixed NVFP4/FP8 (MLP projections NVFP4 group 16; attention/linear-attention projections and last 8 MLP layers FP8 dynamic; AWQ smoothing + GPTQ; FP8 KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "AIME 2025 0-shot (lighteval, vLLM; mean of 8 seeds)",
"full": 0.9667,
"quant_score": 0.95,
"retention": 0.9827247336298748,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/README.md#L173",
"source_id": "quant-redhatai-qwen3-8-27b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 173,
"raw_line": " 95.00% | ",
"row_label_line": 171,
"row_label": "AIME 2025",
"column_header": "RedHatAI/Qwen3.8-27B-NVFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 172,
"raw_line": " 96.67% | ",
"row_label_line": 171,
"row_label": "AIME 2025",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-NVFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. SWE Bench row (Inspect AI; variant not stated) and the separate \"NVFP4 build comparison\" table (unsloth/Inferact builds, no BF16 column) are not used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-NVFP4@d23c6ff:card table",
"artifact": "RedHatAI/Qwen3.8-27B-NVFP4",
"artifact_revision": "d23c6ff198a005532746610e9d719c9f6d27a2b1",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/config.json",
"sha256": "f88aad53f300b6445ca79249c9c98ec45f82576bfec27f7354ba1729d6788776",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "Mixed NVFP4/FP8 (MLP projections NVFP4 group 16; attention/linear-attention projections and last 8 MLP layers FP8 dynamic; AWQ smoothing + GPTQ; FP8 KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "GPQA Diamond 0-shot (lighteval, vLLM; mean of 3 seeds)",
"full": 0.8956,
"quant_score": 0.8922,
"retention": 0.9962036623492632,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/README.md#L179",
"source_id": "quant-redhatai-qwen3-8-27b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 179,
"raw_line": " 89.22% | ",
"row_label_line": 177,
"row_label": "GPQA Diamond",
"column_header": "RedHatAI/Qwen3.8-27B-NVFP4",
"table_category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 178,
"raw_line": " 89.56% | ",
"row_label_line": 177,
"row_label": "GPQA Diamond",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-NVFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. SWE Bench row (Inspect AI; variant not stated) and the separate \"NVFP4 build comparison\" table (unsloth/Inferact builds, no BF16 column) are not used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-NVFP4@d23c6ff:card table",
"artifact": "RedHatAI/Qwen3.8-27B-NVFP4",
"artifact_revision": "d23c6ff198a005532746610e9d719c9f6d27a2b1",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/config.json",
"sha256": "f88aad53f300b6445ca79249c9c98ec45f82576bfec27f7354ba1729d6788776",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "Qwen3.8-27B",
"model_id": "qwen3-8-27b",
"params_b": 27.3,
"quant": "Mixed NVFP4/FP8 (MLP projections NVFP4 group 16; attention/linear-attention projections and last 8 MLP layers FP8 dynamic; AWQ smoothing + GPTQ; FP8 KV cache; lm_head, vision encoder, MTP and linear-attention in_proj_a/b unquantized; llm-compressor)",
"metric": "IFEval 0-shot (lm-evaluation-harness, vLLM; reported metric variant not stated; mean of 3 seeds)",
"full": 0.9119,
"quant_score": 0.9199,
"retention": 1.0087728917644478,
"baseline_quant": "BF16 (unquantized Qwen/Qwen3.8-27B, same Red Hat evaluation table)",
"source": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/README.md#L186",
"source_id": "quant-redhatai-qwen3-8-27b-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 186,
"raw_line": " 91.99% | ",
"row_label_line": 184,
"row_label": "IFEval",
"column_header": "RedHatAI/Qwen3.8-27B-NVFP4",
"table_category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/README.md",
"sha256": "24446de30e61f3443195fe924dc97884025c624bf881a1704a8d4758b328a310",
"line": 185,
"raw_line": " 91.19% | ",
"row_label_line": 184,
"row_label": "IFEval",
"column_header": "Qwen/Qwen3.8-27B",
"table_category": "Instruction Following"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed, not copied from the card's Recovery column",
"note": "Red Hat (Neural Magic) evaluation published in the pinned RedHatAI/Qwen3.8-27B-NVFP4 model card; baseline column \"Qwen/Qwen3.8-27B\" and quantized column \"RedHatAI/Qwen3.8-27B-NVFP4\" are from the same card table. Red Hat evaluation: vLLM OpenAI-compatible server with --reasoning-parser qwen3 (thinking-mode default); lm-evaluation-harness (neuralmagic fork) for GSM8K Platinum/IFEval/MMLU-Pro and lighteval (neuralmagic fork) for MATH-500/GPQA Diamond/AIME 2025; 3 seeds (AIME 2025: 8 seeds), mean reported (per card). Evaluations run on 1x B200; sampling temperature 1.0, top_p 0.95, top_k 20, max 65536 new tokens. SWE Bench row (Inspect AI; variant not stated) and the separate \"NVFP4 build comparison\" table (unsloth/Inferact builds, no BF16 column) are not used. Relative within-study evidence only; absolute scores are harness/prompt specific and not comparable with vendor-reported numbers.",
"aggregation": "single benchmark",
"study": "RedHatAI/Qwen3.8-27B-NVFP4@d23c6ff:card table",
"artifact": "RedHatAI/Qwen3.8-27B-NVFP4",
"artifact_revision": "d23c6ff198a005532746610e9d719c9f6d27a2b1",
"quant_config_evidence": {
"file": "speed-evidence/RedHatAI__Qwen3.8-27B-NVFP4/config.json",
"sha256": "f88aad53f300b6445ca79249c9c98ec45f82576bfec27f7354ba1729d6788776",
"url": "https://huggingface.co/RedHatAI/Qwen3.8-27B-NVFP4/blob/d23c6ff198a005532746610e9d719c9f6d27a2b1/config.json",
"use": "quantization recipe label only (quantization_config)"
}
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.5003,
"quant_score": 0.5017,
"retention": 1.0027983210073956,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L173",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 173,
"raw_line": "50.17 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 172,
"raw_line": "50.03 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 100.28
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "AIME 2024 pass@1",
"full": 0.3005,
"quant_score": 0.2983,
"retention": 0.9926788685524127,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L155",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 155,
"raw_line": "29.83 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 154,
"raw_line": "30.05 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 99.27
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MATH-500 pass@1",
"full": 0.8466,
"quant_score": 0.8474,
"retention": 1.0009449562957713,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L161",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 161,
"raw_line": "84.74 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 160,
"raw_line": "84.66 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 100.09
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GPQA Diamond pass@1",
"full": 0.3537,
"quant_score": 0.3593,
"retention": 1.0158326265196496,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L167",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 167,
"raw_line": "35.93 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 166,
"raw_line": "35.37 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 101.58
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.4799,
"quant_score": 0.4782,
"retention": 0.996457595332361,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L216",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 216,
"raw_line": " 47.82 | ",
"row_label_line": 214,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 215,
"raw_line": " 47.99 | ",
"row_label_line": 214,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 99.7
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.6998,
"quant_score": 0.6899,
"retention": 0.9858531008859672,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L186",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 186,
"raw_line": " 68.99 | ",
"row_label_line": 184,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 185,
"raw_line": " 69.98 | ",
"row_label_line": 184,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 98.6
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.3738,
"quant_score": 0.3722,
"retention": 0.9957196361690742,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L198",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 198,
"raw_line": " 37.22 | ",
"row_label_line": 196,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 197,
"raw_line": " 37.38 | ",
"row_label_line": 196,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 99.6
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.3463,
"quant_score": 0.3491,
"retention": 1.0080854750216572,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/blob/b87f998fcda08540b81b50844bcdb811540db5e8/README.md#L223",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 223,
"raw_line": " 34.91 | ",
"row_label_line": 221,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic/README.md",
"sha256": "3025871cf2cfc6808983700fe9772d69fef47d0e8699f9b9d9057e0f3bb13a85",
"line": 222,
"raw_line": " 34.63 | ",
"row_label_line": 221,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic@b87f998:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-FP8-dynamic",
"artifact_revision": "b87f998fcda08540b81b50844bcdb811540db5e8",
"published_recovery_pct": 100.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.5003,
"quant_score": 0.4677,
"retention": 0.9348390965420749,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/blob/ececd8aa3863503391260ef30bec33887694dea0/README.md#L188",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 188,
"raw_line": "46.77 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 187,
"raw_line": "50.03 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16@ececd8a:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"artifact_revision": "ececd8aa3863503391260ef30bec33887694dea0",
"published_recovery_pct": 93.48
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.3005,
"quant_score": 0.2463,
"retention": 0.8196339434276206,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/blob/ececd8aa3863503391260ef30bec33887694dea0/README.md#L170",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 170,
"raw_line": "24.63 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 169,
"raw_line": "30.05 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16@ececd8a:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"artifact_revision": "ececd8aa3863503391260ef30bec33887694dea0",
"published_recovery_pct": 81.96
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.3537,
"quant_score": 0.3316,
"retention": 0.9375176703420978,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/blob/ececd8aa3863503391260ef30bec33887694dea0/README.md#L182",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 182,
"raw_line": "33.16 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 181,
"raw_line": "35.37 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16@ececd8a:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"artifact_revision": "ececd8aa3863503391260ef30bec33887694dea0",
"published_recovery_pct": 93.75
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.4799,
"quant_score": 0.4756,
"retention": 0.9910397999583247,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/blob/ececd8aa3863503391260ef30bec33887694dea0/README.md#L231",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 231,
"raw_line": " 47.56 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 230,
"raw_line": " 47.99 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16@ececd8a:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"artifact_revision": "ececd8aa3863503391260ef30bec33887694dea0",
"published_recovery_pct": 99.1
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.6998,
"quant_score": 0.6801,
"retention": 0.9718490997427836,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/blob/ececd8aa3863503391260ef30bec33887694dea0/README.md#L201",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 201,
"raw_line": " 68.01 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 200,
"raw_line": " 69.98 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16@ececd8a:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"artifact_revision": "ececd8aa3863503391260ef30bec33887694dea0",
"published_recovery_pct": 97.2
},
{
"model": "DeepSeek-R1-Distill-Qwen-1.5B",
"model_id": "deepseek-r1-distill-qwen-1-5b",
"params_b": 1.78,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.3738,
"quant_score": 0.3698,
"retention": 0.9892990904226858,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/blob/ececd8aa3863503391260ef30bec33887694dea0/README.md#L213",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-1-5b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 213,
"raw_line": " 36.98 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16/README.md",
"sha256": "e1b9a84d1e5dc0fa4ff30a145dfd39430c6d6993065c95b829851e8650b01a43",
"line": 212,
"raw_line": " 37.38 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16@ececd8a:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-1.5B-quantized.w4a16",
"artifact_revision": "ececd8aa3863503391260ef30bec33887694dea0",
"published_recovery_pct": 98.9
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.6579,
"quant_score": 0.6569,
"retention": 0.9984800121599026,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/blob/ceb2fcd178d477cd21b92ffb43164c74165a212c/README.md#L173",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 173,
"raw_line": "65.69 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 172,
"raw_line": "65.79 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic@ceb2fcd:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"artifact_revision": "ceb2fcd178d477cd21b92ffb43164c74165a212c",
"published_recovery_pct": 99.85
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "AIME 2024 pass@1",
"full": 0.5317,
"quant_score": 0.5317,
"retention": 1,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/blob/ceb2fcd178d477cd21b92ffb43164c74165a212c/README.md#L155",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 155,
"raw_line": "53.17 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 154,
"raw_line": "53.17 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic@ceb2fcd:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"artifact_revision": "ceb2fcd178d477cd21b92ffb43164c74165a212c",
"published_recovery_pct": 100
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MATH-500 pass@1",
"full": 0.9366,
"quant_score": 0.9362,
"retention": 0.9995729233397396,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/blob/ceb2fcd178d477cd21b92ffb43164c74165a212c/README.md#L161",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 161,
"raw_line": "93.62 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 160,
"raw_line": "93.66 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic@ceb2fcd:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"artifact_revision": "ceb2fcd178d477cd21b92ffb43164c74165a212c",
"published_recovery_pct": 99.96
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GPQA Diamond pass@1",
"full": 0.5053,
"quant_score": 0.5028,
"retention": 0.9950524440926184,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/blob/ceb2fcd178d477cd21b92ffb43164c74165a212c/README.md#L167",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 167,
"raw_line": "50.28 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 166,
"raw_line": "50.53 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic@ceb2fcd:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"artifact_revision": "ceb2fcd178d477cd21b92ffb43164c74165a212c",
"published_recovery_pct": 99.51
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.3938,
"quant_score": 0.3901,
"retention": 0.9906043676993397,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/blob/ceb2fcd178d477cd21b92ffb43164c74165a212c/README.md#L223",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 223,
"raw_line": " 39.01 | ",
"row_label_line": 221,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic/README.md",
"sha256": "563adc6f84ba501e306d9beabfcde75158855cd2bc0ed2955fbb99b7bf30b246",
"line": 222,
"raw_line": " 39.38 | ",
"row_label_line": 221,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic@ceb2fcd:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-FP8-dynamic",
"artifact_revision": "ceb2fcd178d477cd21b92ffb43164c74165a212c",
"published_recovery_pct": 99.1
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.6579,
"quant_score": 0.6468,
"retention": 0.9831281349749202,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/blob/83b68401b8a9f9439725acc32a6e9c350451fb64/README.md#L188",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 188,
"raw_line": "64.68 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 187,
"raw_line": "65.79 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16@83b6840:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"artifact_revision": "83b68401b8a9f9439725acc32a6e9c350451fb64",
"published_recovery_pct": 98.31
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.5317,
"quant_score": 0.5088,
"retention": 0.9569305999623847,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/blob/83b68401b8a9f9439725acc32a6e9c350451fb64/README.md#L170",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 170,
"raw_line": "50.88 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 169,
"raw_line": "53.17 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16@83b6840:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"artifact_revision": "83b68401b8a9f9439725acc32a6e9c350451fb64",
"published_recovery_pct": 95.69
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9366,
"quant_score": 0.9333,
"retention": 0.9964766175528508,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/blob/83b68401b8a9f9439725acc32a6e9c350451fb64/README.md#L176",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 176,
"raw_line": "93.33 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 175,
"raw_line": "93.66 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16@83b6840:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"artifact_revision": "83b68401b8a9f9439725acc32a6e9c350451fb64",
"published_recovery_pct": 99.65
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.5053,
"quant_score": 0.4984,
"retention": 0.9863447456956265,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/blob/83b68401b8a9f9439725acc32a6e9c350451fb64/README.md#L182",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 182,
"raw_line": "49.84 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 181,
"raw_line": "50.53 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16@83b6840:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"artifact_revision": "83b68401b8a9f9439725acc32a6e9c350451fb64",
"published_recovery_pct": 98.63
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.3938,
"quant_score": 0.3938,
"retention": 1,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/blob/83b68401b8a9f9439725acc32a6e9c350451fb64/README.md#L238",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 238,
"raw_line": " 39.38 | ",
"row_label_line": 236,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16/README.md",
"sha256": "c961f717c815488a405c6e10095bf4448c1e3742716446c8e4ab8e8461149f57",
"line": 237,
"raw_line": " 39.38 | ",
"row_label_line": 236,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16@83b6840:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w4a16",
"artifact_revision": "83b68401b8a9f9439725acc32a6e9c350451fb64",
"published_recovery_pct": 100
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.6579,
"quant_score": 0.6628,
"retention": 1.0074479404164767,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/blob/925259de674c80799e9455bfcd2bf50f999022cb/README.md#L192",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 192,
"raw_line": "66.28 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 191,
"raw_line": "65.79 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8@925259d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"artifact_revision": "925259de674c80799e9455bfcd2bf50f999022cb",
"published_recovery_pct": 100.74
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.5317,
"quant_score": 0.5519,
"retention": 1.0379913485047958,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/blob/925259de674c80799e9455bfcd2bf50f999022cb/README.md#L174",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 174,
"raw_line": "55.19 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 173,
"raw_line": "53.17 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8@925259d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"artifact_revision": "925259de674c80799e9455bfcd2bf50f999022cb",
"published_recovery_pct": 103.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9366,
"quant_score": 0.93,
"retention": 0.9929532351057015,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/blob/925259de674c80799e9455bfcd2bf50f999022cb/README.md#L180",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 180,
"raw_line": "93 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 179,
"raw_line": "93.66 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8@925259d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"artifact_revision": "925259de674c80799e9455bfcd2bf50f999022cb",
"published_recovery_pct": 99.3
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.5053,
"quant_score": 0.5065,
"retention": 1.0023748268355432,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/blob/925259de674c80799e9455bfcd2bf50f999022cb/README.md#L186",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 186,
"raw_line": "50.65 | ",
"row_label_line": 184,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 185,
"raw_line": "50.53 | ",
"row_label_line": 184,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8@925259d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"artifact_revision": "925259de674c80799e9455bfcd2bf50f999022cb",
"published_recovery_pct": 100.24
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.3938,
"quant_score": 0.402,
"retention": 1.020822752666328,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/blob/925259de674c80799e9455bfcd2bf50f999022cb/README.md#L242",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-7b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 242,
"raw_line": " 40.20 | ",
"row_label_line": 240,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8/README.md",
"sha256": "633d6a4f71c481da99d71c649f611e0eff5d54ab655786d9af5f4f6c09392104",
"line": 241,
"raw_line": " 39.38 | ",
"row_label_line": 240,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8@925259d:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-7B-quantized.w8a8",
"artifact_revision": "925259de674c80799e9455bfcd2bf50f999022cb",
"published_recovery_pct": 102.1
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.7356,
"quant_score": 0.7429,
"retention": 1.0099238716693857,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L173",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 173,
"raw_line": "74.29 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 172,
"raw_line": "73.56 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 100.99
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "AIME 2024 pass@1",
"full": 0.6667,
"quant_score": 0.6813,
"retention": 1.0218989050547471,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L155",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 155,
"raw_line": "68.13 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 154,
"raw_line": "66.67 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 102.19
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MATH-500 pass@1",
"full": 0.9466,
"quant_score": 0.9463,
"retention": 0.9996830762729769,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L161",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 161,
"raw_line": "94.63 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 160,
"raw_line": "94.66 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 99.97
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GPQA Diamond pass@1",
"full": 0.5935,
"quant_score": 0.601,
"retention": 1.012636899747262,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L167",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 167,
"raw_line": "60.1 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 166,
"raw_line": "59.35 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 101.26
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7099,
"quant_score": 0.7081,
"retention": 0.997464431610086,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L216",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 216,
"raw_line": " 70.81 | ",
"row_label_line": 214,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 215,
"raw_line": " 70.99 | ",
"row_label_line": 214,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8704,
"quant_score": 0.8741,
"retention": 1.004250919117647,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L186",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 186,
"raw_line": " 87.41 | ",
"row_label_line": 184,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 185,
"raw_line": " 87.04 | ",
"row_label_line": 184,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 100.4
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7446,
"quant_score": 0.7463,
"retention": 1.0022831050228311,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L198",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 198,
"raw_line": " 74.63 | ",
"row_label_line": 196,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 197,
"raw_line": " 74.46 | ",
"row_label_line": 196,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 100.2
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.3421,
"quant_score": 0.3439,
"retention": 1.0052616194095292,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L259",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 259,
"raw_line": " 34.39 | ",
"row_label_line": 257,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 258,
"raw_line": " 34.21 | ",
"row_label_line": 257,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 100.5
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.4305,
"quant_score": 0.4369,
"retention": 1.0148664343786293,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L223",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 223,
"raw_line": " 43.69 | ",
"row_label_line": 221,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 222,
"raw_line": " 43.05 | ",
"row_label_line": 221,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 101.5
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published, normalisation not stated)",
"full": 0.3486,
"quant_score": 0.3504,
"retention": 1.0051635111876074,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/blob/cdf473db71c978fcf36cbdacd44da74ebd055354/README.md#L253",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 253,
"raw_line": " 35.04 | ",
"row_label_line": 251,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic/README.md",
"sha256": "13674b9d66177ce878bf6968a26d76c8bbc79e8156a63f5b0de187d2742943e8",
"line": 252,
"raw_line": " 34.86 | ",
"row_label_line": 251,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic@cdf473d:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-FP8-dynamic",
"artifact_revision": "cdf473db71c978fcf36cbdacd44da74ebd055354",
"published_recovery_pct": 100.5
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.7356,
"quant_score": 0.7282,
"retention": 0.9899401848830885,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L188",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 188,
"raw_line": "72.82 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 187,
"raw_line": "73.56 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 98.99
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.6667,
"quant_score": 0.6604,
"retention": 0.9905504724763762,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L170",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 170,
"raw_line": "66.04 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 169,
"raw_line": "66.67 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 99.06
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9466,
"quant_score": 0.9495,
"retention": 1.0030635960278893,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L176",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 176,
"raw_line": "94.95 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 175,
"raw_line": "94.66 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 100.31
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.5935,
"quant_score": 0.5748,
"retention": 0.96849199663016,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L182",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 182,
"raw_line": "57.48 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 181,
"raw_line": "59.35 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 96.85
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7099,
"quant_score": 0.7085,
"retention": 0.998027891252289,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L231",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 231,
"raw_line": " 70.85 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 230,
"raw_line": " 70.99 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8704,
"quant_score": 0.8734,
"retention": 1.0034466911764706,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L201",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 201,
"raw_line": " 87.34 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 200,
"raw_line": " 87.04 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 100.4
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7446,
"quant_score": 0.7332,
"retention": 0.984689766317486,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L213",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 213,
"raw_line": " 73.32 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 212,
"raw_line": " 74.46 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 98.5
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.3421,
"quant_score": 0.3241,
"retention": 0.9473838059047061,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L274",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 274,
"raw_line": " 32.41 | ",
"row_label_line": 272,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 273,
"raw_line": " 34.21 | ",
"row_label_line": 272,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 94.7
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.4305,
"quant_score": 0.349,
"retention": 0.81068524970964,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L238",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 238,
"raw_line": " 34.90 | ",
"row_label_line": 236,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 237,
"raw_line": " 43.05 | ",
"row_label_line": 236,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 81.1
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published, normalisation not stated)",
"full": 0.3486,
"quant_score": 0.3509,
"retention": 1.006597819850832,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/blob/90ab462f58256472697848376961c8a5b754b094/README.md#L268",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 268,
"raw_line": " 35.09 | ",
"row_label_line": 266,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16/README.md",
"sha256": "6b7e2e5e0710126d2f7339dc81e51f374bd1d93fc472cee8790234c3abe0f72a",
"line": 267,
"raw_line": " 34.86 | ",
"row_label_line": 266,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16@90ab462:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w4a16",
"artifact_revision": "90ab462f58256472697848376961c8a5b754b094",
"published_recovery_pct": 100.7
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.7356,
"quant_score": 0.731,
"retention": 0.9937466014138118,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/blob/52aeeeebff0c6a91a04a2971df4e106dbaf94966/README.md#L192",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 192,
"raw_line": "73.1 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 191,
"raw_line": "73.56 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8@52aeeee:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"artifact_revision": "52aeeeebff0c6a91a04a2971df4e106dbaf94966",
"published_recovery_pct": 99.37
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.6667,
"quant_score": 0.6631,
"retention": 0.9946002699865006,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/blob/52aeeeebff0c6a91a04a2971df4e106dbaf94966/README.md#L174",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 174,
"raw_line": "66.31 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 173,
"raw_line": "66.67 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8@52aeeee:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"artifact_revision": "52aeeeebff0c6a91a04a2971df4e106dbaf94966",
"published_recovery_pct": 99.46
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9466,
"quant_score": 0.9468,
"retention": 1.000211282484682,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/blob/52aeeeebff0c6a91a04a2971df4e106dbaf94966/README.md#L180",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 180,
"raw_line": "94.68 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 179,
"raw_line": "94.66 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8@52aeeee:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"artifact_revision": "52aeeeebff0c6a91a04a2971df4e106dbaf94966",
"published_recovery_pct": 100.02
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.5935,
"quant_score": 0.5832,
"retention": 0.9826453243470935,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/blob/52aeeeebff0c6a91a04a2971df4e106dbaf94966/README.md#L186",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 186,
"raw_line": "58.32 | ",
"row_label_line": 184,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 185,
"raw_line": "59.35 | ",
"row_label_line": 184,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8@52aeeee:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"artifact_revision": "52aeeeebff0c6a91a04a2971df4e106dbaf94966",
"published_recovery_pct": 98.26
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7099,
"quant_score": 0.7106,
"retention": 1.0009860543738556,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/blob/52aeeeebff0c6a91a04a2971df4e106dbaf94966/README.md#L235",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 235,
"raw_line": " 71.06 | ",
"row_label_line": 233,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 234,
"raw_line": " 70.99 | ",
"row_label_line": 233,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8@52aeeee:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"artifact_revision": "52aeeeebff0c6a91a04a2971df4e106dbaf94966",
"published_recovery_pct": 100.1
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8704,
"quant_score": 0.8779,
"retention": 1.0086167279411764,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/blob/52aeeeebff0c6a91a04a2971df4e106dbaf94966/README.md#L205",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 205,
"raw_line": " 87.79 | ",
"row_label_line": 203,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 204,
"raw_line": " 87.04 | ",
"row_label_line": 203,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8@52aeeee:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"artifact_revision": "52aeeeebff0c6a91a04a2971df4e106dbaf94966",
"published_recovery_pct": 100.9
},
{
"model": "DeepSeek-R1-Distill-Qwen-14B",
"model_id": "deepseek-r1-distill-qwen-14b",
"params_b": 14.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7446,
"quant_score": 0.7426,
"retention": 0.9973139940907871,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-14B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/blob/52aeeeebff0c6a91a04a2971df4e106dbaf94966/README.md#L217",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-14b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 217,
"raw_line": " 74.26 | ",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8/README.md",
"sha256": "681925d6c2adca8f72c54bb607e14f5bd95533bfc026b9af8fe8e7353634e8f1",
"line": 216,
"raw_line": " 74.46 | ",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8@52aeeee:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-14B-quantized.w8a8",
"artifact_revision": "52aeeeebff0c6a91a04a2971df4e106dbaf94966",
"published_recovery_pct": 99.7
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.763,
"quant_score": 0.7555,
"retention": 0.9901703800786369,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L173",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 173,
"raw_line": "75.55 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 172,
"raw_line": "76.3 | ",
"row_label_line": 171,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 99.02
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "AIME 2024 pass@1",
"full": 0.6975,
"quant_score": 0.685,
"retention": 0.982078853046595,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L155",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 155,
"raw_line": "68.5 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 154,
"raw_line": "69.75 | ",
"row_label_line": 153,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 98.21
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MATH-500 pass@1",
"full": 0.9509,
"quant_score": 0.9526,
"retention": 1.0017877799978967,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L161",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 161,
"raw_line": "95.26 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 160,
"raw_line": "95.09 | ",
"row_label_line": 159,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 100.18
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GPQA Diamond pass@1",
"full": 0.6405,
"quant_score": 0.6288,
"retention": 0.9817330210772834,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L167",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 167,
"raw_line": "62.88 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 166,
"raw_line": "64.05 | ",
"row_label_line": 165,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 98.17
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7451,
"quant_score": 0.7436,
"retention": 0.9979868474030331,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L216",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 216,
"raw_line": " 74.36 | ",
"row_label_line": 214,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 215,
"raw_line": " 74.51 | ",
"row_label_line": 214,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8271,
"quant_score": 0.8264,
"retention": 0.9991536694474672,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L186",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 186,
"raw_line": " 82.64 | ",
"row_label_line": 184,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 185,
"raw_line": " 82.71 | ",
"row_label_line": 184,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 99.9
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.8112,
"quant_score": 0.8098,
"retention": 0.9982741617357003,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L198",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 198,
"raw_line": " 80.98 | ",
"row_label_line": 196,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 197,
"raw_line": " 81.12 | ",
"row_label_line": 196,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.3692,
"quant_score": 0.3686,
"retention": 0.9983748645720476,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L259",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 259,
"raw_line": " 36.86 | ",
"row_label_line": 257,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 258,
"raw_line": " 36.92 | ",
"row_label_line": 257,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published, normalisation not stated)",
"full": 0.4982,
"quant_score": 0.4943,
"retention": 0.9921718185467685,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/blob/e702fff1b22be372c8893745aa9a13cf1a5d0d8e/README.md#L253",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 253,
"raw_line": " 49.43 | ",
"row_label_line": 251,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic/README.md",
"sha256": "69c818b3a1eef84bd467b8eca040f271ce777e4f3b8b4a3115af96221c9078e9",
"line": 252,
"raw_line": " 49.82 | ",
"row_label_line": 251,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic@e702fff:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-FP8-dynamic",
"artifact_revision": "e702fff1b22be372c8893745aa9a13cf1a5d0d8e",
"published_recovery_pct": 99.2
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.763,
"quant_score": 0.7588,
"retention": 0.9944954128440365,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L188",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 188,
"raw_line": "75.88 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 187,
"raw_line": "76.3 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 99.45
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.6975,
"quant_score": 0.6883,
"retention": 0.9868100358422939,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L170",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 170,
"raw_line": "68.83 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 169,
"raw_line": "69.75 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 98.68
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9509,
"quant_score": 0.9501,
"retention": 0.9991586917656956,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L176",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 176,
"raw_line": "95.01 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 175,
"raw_line": "95.09 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 99.92
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.6405,
"quant_score": 0.6379,
"retention": 0.9959406713505076,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L182",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 182,
"raw_line": "63.79 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 181,
"raw_line": "64.05 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 99.59
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7451,
"quant_score": 0.7432,
"retention": 0.9974500067105085,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L231",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 231,
"raw_line": " 74.32 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 230,
"raw_line": " 74.51 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8271,
"quant_score": 0.8438,
"retention": 1.020191028896143,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L201",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 201,
"raw_line": " 84.38 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 200,
"raw_line": " 82.71 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 102
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.8112,
"quant_score": 0.8091,
"retention": 0.9974112426035502,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L213",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 213,
"raw_line": " 80.91 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 212,
"raw_line": " 81.12 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 99.7
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published, normalisation not stated)",
"full": 0.4982,
"quant_score": 0.4843,
"retention": 0.9720995584102771,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/blob/f3572d23e9bc8dc194f361cfaf9ab73383e9125a/README.md#L268",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 268,
"raw_line": " 48.43 | ",
"row_label_line": 266,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16/README.md",
"sha256": "3b75d647fdaaff0f8870fc3296f538c6bf30c367cceb3b1e9c09a644a813537d",
"line": 267,
"raw_line": " 49.82 | ",
"row_label_line": 266,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16@f3572d2:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w4a16",
"artifact_revision": "f3572d23e9bc8dc194f361cfaf9ab73383e9125a",
"published_recovery_pct": 97.2
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.763,
"quant_score": 0.7597,
"retention": 0.9956749672346002,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L192",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 192,
"raw_line": "75.97 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 191,
"raw_line": "76.3 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 99.57
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.6975,
"quant_score": 0.6817,
"retention": 0.977347670250896,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L174",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 174,
"raw_line": "68.17 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 173,
"raw_line": "69.75 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 97.73
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9509,
"quant_score": 0.9498,
"retention": 0.9988432011778315,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L180",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 180,
"raw_line": "94.98 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 179,
"raw_line": "95.09 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 99.88
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.6405,
"quant_score": 0.6475,
"retention": 1.010928961748634,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L186",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 186,
"raw_line": "64.75 | ",
"row_label_line": 184,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 185,
"raw_line": "64.05 | ",
"row_label_line": 184,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 101.09
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7451,
"quant_score": 0.745,
"retention": 0.9998657898268687,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L235",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 235,
"raw_line": " 74.50 | ",
"row_label_line": 233,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 234,
"raw_line": " 74.51 | ",
"row_label_line": 233,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 100
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8271,
"quant_score": 0.8385,
"retention": 1.0137830975698223,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L205",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 205,
"raw_line": " 83.85 | ",
"row_label_line": 203,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 204,
"raw_line": " 82.71 | ",
"row_label_line": 203,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 101.4
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.8112,
"quant_score": 0.8094,
"retention": 0.9977810650887574,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L217",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 217,
"raw_line": " 80.94 | ",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 216,
"raw_line": " 81.12 | ",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.3692,
"quant_score": 0.37,
"retention": 1.0021668472372698,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L278",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 278,
"raw_line": " 37.00 | ",
"row_label_line": 276,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 277,
"raw_line": " 36.92 | ",
"row_label_line": 276,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 100.2
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.4287,
"quant_score": 0.4192,
"retention": 0.9778399813389318,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L242",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 242,
"raw_line": " 41.92 | ",
"row_label_line": 240,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 241,
"raw_line": " 42.87 | ",
"row_label_line": 240,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 97.8
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published, normalisation not stated)",
"full": 0.4982,
"quant_score": 0.4914,
"retention": 0.9863508631071859,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Qwen-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/blob/bfc01211470299398548c5d6ff8ca2e48dc9a3dc/README.md#L272",
"source_id": "quant-redhatai-deepseek-r1-distill-qwen-32b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 272,
"raw_line": " 49.14 | ",
"row_label_line": 270,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8/README.md",
"sha256": "575a5c92773d27c4fe0a1ca5106596e57a0967bb4e1352db18b44dc2801604b5",
"line": 271,
"raw_line": " 49.82 | ",
"row_label_line": 270,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8@bfc0121:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Qwen-32B-quantized.w8a8",
"artifact_revision": "bfc01211470299398548c5d6ff8ca2e48dc9a3dc",
"published_recovery_pct": 98.6
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.629,
"quant_score": 0.6326,
"retention": 1.0057233704292527,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/blob/5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd/README.md#L172",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 172,
"raw_line": " 63.26 | ",
"row_label_line": 170,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 171,
"raw_line": " 62.9 | ",
"row_label_line": 170,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic@5d548d9:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"artifact_revision": "5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd",
"published_recovery_pct": 100.57
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "AIME 2024 pass@1",
"full": 0.4925,
"quant_score": 0.5083,
"retention": 1.0320812182741117,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/blob/5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd/README.md#L154",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 154,
"raw_line": " 50.83 | ",
"row_label_line": 152,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 153,
"raw_line": " 49.25 | ",
"row_label_line": 152,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic@5d548d9:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"artifact_revision": "5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd",
"published_recovery_pct": 103.21
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MATH-500 pass@1",
"full": 0.9018,
"quant_score": 0.9024,
"retention": 1.0006653359946773,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/blob/5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd/README.md#L160",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 160,
"raw_line": " 90.24 | ",
"row_label_line": 158,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 159,
"raw_line": " 90.18 | ",
"row_label_line": 158,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic@5d548d9:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"artifact_revision": "5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd",
"published_recovery_pct": 100.07
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GPQA Diamond pass@1",
"full": 0.4927,
"quant_score": 0.4871,
"retention": 0.9886340572356404,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/blob/5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd/README.md#L166",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 166,
"raw_line": " 48.71 | ",
"row_label_line": 164,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 165,
"raw_line": " 49.27 | ",
"row_label_line": 164,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic@5d548d9:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"artifact_revision": "5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd",
"published_recovery_pct": 98.86
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.3837,
"quant_score": 0.3867,
"retention": 1.007818608287725,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/blob/5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd/README.md#L222",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 222,
"raw_line": " 38.67 | ",
"row_label_line": 220,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-FP8-dynamic/README.md",
"sha256": "55fa2095573c45c01faeb831cfea7419b7d467cc3f7692e45699026e400057e3",
"line": 221,
"raw_line": " 38.37 | ",
"row_label_line": 220,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic@5d548d9:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-FP8-dynamic",
"artifact_revision": "5d548d9169c53b7bb7ef0b3bc261509d5da6e3dd",
"published_recovery_pct": 100.8
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.629,
"quant_score": 0.6112,
"retention": 0.9717011128775834,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L188",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 188,
"raw_line": "61.12 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 187,
"raw_line": "62.9 | ",
"row_label_line": 186,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 97.17
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.4925,
"quant_score": 0.463,
"retention": 0.9401015228426396,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L170",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 170,
"raw_line": "46.3 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 169,
"raw_line": "49.25 | ",
"row_label_line": 168,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 94.01
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9018,
"quant_score": 0.8994,
"retention": 0.9973386560212907,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L176",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 176,
"raw_line": "89.94 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 175,
"raw_line": "90.18 | ",
"row_label_line": 174,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 99.73
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.4927,
"quant_score": 0.4711,
"retention": 0.9561599350517557,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L182",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 182,
"raw_line": "47.11 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 181,
"raw_line": "49.27 | ",
"row_label_line": 180,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 95.62
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.5988,
"quant_score": 0.5829,
"retention": 0.9734468937875751,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L231",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 231,
"raw_line": " 58.29 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 230,
"raw_line": " 59.88 | ",
"row_label_line": 229,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 97.3
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.6277,
"quant_score": 0.6308,
"retention": 1.004938664967341,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L201",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 201,
"raw_line": " 63.08 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 200,
"raw_line": " 62.77 | ",
"row_label_line": 199,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 100.5
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.5565,
"quant_score": 0.5211,
"retention": 0.9363881401617251,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L213",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 213,
"raw_line": " 52.11 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 212,
"raw_line": " 55.65 | ",
"row_label_line": 211,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 93.6
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.3837,
"quant_score": 0.3785,
"retention": 0.9864477456346105,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/blob/2ed7bf0c3a423fd114e70a86f7e30420870709af/README.md#L238",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 238,
"raw_line": " 37.85 | ",
"row_label_line": 236,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w4a16/README.md",
"sha256": "c47177bc48e7310b1258cb302195bef4bbfe5e34b697059bf1e3f5e762ac7c9e",
"line": 237,
"raw_line": " 38.37 | ",
"row_label_line": 236,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16@2ed7bf0:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w4a16",
"artifact_revision": "2ed7bf0c3a423fd114e70a86f7e30420870709af",
"published_recovery_pct": 98.7
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.629,
"quant_score": 0.6267,
"retention": 0.9963434022257552,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/blob/d167aa19e86a90b8f205cebd68e986a72c76be57/README.md#L192",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 192,
"raw_line": " 62.67 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 191,
"raw_line": " 62.9 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8@d167aa1:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"artifact_revision": "d167aa19e86a90b8f205cebd68e986a72c76be57",
"published_recovery_pct": 99.63
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.4925,
"quant_score": 0.4907,
"retention": 0.9963451776649747,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/blob/d167aa19e86a90b8f205cebd68e986a72c76be57/README.md#L174",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 174,
"raw_line": " 49.07 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 173,
"raw_line": " 49.25 | ",
"row_label_line": 172,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8@d167aa1:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"artifact_revision": "d167aa19e86a90b8f205cebd68e986a72c76be57",
"published_recovery_pct": 99.63
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9018,
"quant_score": 0.9004,
"retention": 0.998447549345753,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/blob/d167aa19e86a90b8f205cebd68e986a72c76be57/README.md#L180",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 180,
"raw_line": " 90.04 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 179,
"raw_line": " 90.18 | ",
"row_label_line": 178,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8@d167aa1:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"artifact_revision": "d167aa19e86a90b8f205cebd68e986a72c76be57",
"published_recovery_pct": 99.84
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.5988,
"quant_score": 0.5949,
"retention": 0.9934869739478958,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/blob/d167aa19e86a90b8f205cebd68e986a72c76be57/README.md#L235",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 235,
"raw_line": " 59.49 | ",
"row_label_line": 233,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 234,
"raw_line": " 59.88 | ",
"row_label_line": 233,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8@d167aa1:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"artifact_revision": "d167aa19e86a90b8f205cebd68e986a72c76be57",
"published_recovery_pct": 99.3
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.6277,
"quant_score": 0.6209,
"retention": 0.9891667994264776,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/blob/d167aa19e86a90b8f205cebd68e986a72c76be57/README.md#L205",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 205,
"raw_line": " 62.09 | ",
"row_label_line": 203,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 204,
"raw_line": " 62.77 | ",
"row_label_line": 203,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8@d167aa1:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"artifact_revision": "d167aa19e86a90b8f205cebd68e986a72c76be57",
"published_recovery_pct": 98.9
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.5565,
"quant_score": 0.5553,
"retention": 0.997843665768194,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/blob/d167aa19e86a90b8f205cebd68e986a72c76be57/README.md#L217",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 217,
"raw_line": " 55.53 | ",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 216,
"raw_line": " 55.65 | ",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8@d167aa1:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"artifact_revision": "d167aa19e86a90b8f205cebd68e986a72c76be57",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.3837,
"quant_score": 0.384,
"retention": 1.0007818608287726,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-8B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/blob/d167aa19e86a90b8f205cebd68e986a72c76be57/README.md#L242",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-8b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 242,
"raw_line": " 38.40 | ",
"row_label_line": 240,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-8B-quantized.w8a8/README.md",
"sha256": "3a1d83146a42acfe69c4576fb556247d3f6a046b9ca8ce6a9d2c300146e0c8cc",
"line": 241,
"raw_line": " 38.37 | ",
"row_label_line": 240,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8@d167aa1:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-8B-quantized.w8a8",
"artifact_revision": "d167aa19e86a90b8f205cebd68e986a72c76be57",
"published_recovery_pct": 100.1
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.7623,
"quant_score": 0.7649,
"retention": 1.0034107306834577,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L182",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 182,
"raw_line": "76.49 | ",
"row_label_line": 180,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 181,
"raw_line": "76.23 | ",
"row_label_line": 180,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 100.34
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "AIME 2024 pass@1",
"full": 0.6783,
"quant_score": 0.6917,
"retention": 1.0197552705292643,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L164",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 164,
"raw_line": "69.17 | ",
"row_label_line": 162,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 163,
"raw_line": "67.83 | ",
"row_label_line": 162,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 101.98
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MATH-500 pass@1",
"full": 0.9529,
"quant_score": 0.9514,
"retention": 0.9984258579074404,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L170",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 170,
"raw_line": "95.14 | ",
"row_label_line": 168,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 169,
"raw_line": "95.29 | ",
"row_label_line": 168,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 99.84
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GPQA Diamond pass@1",
"full": 0.6557,
"quant_score": 0.6515,
"retention": 0.9935946316913224,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L176",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 176,
"raw_line": "65.15 | ",
"row_label_line": 174,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 175,
"raw_line": "65.57 | ",
"row_label_line": 174,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 99.36
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7574,
"quant_score": 0.7555,
"retention": 0.9974914180089781,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L225",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 225,
"raw_line": " 75.55 | ",
"row_label_line": 223,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 224,
"raw_line": " 75.74 | ",
"row_label_line": 223,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 99.8
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.9303,
"quant_score": 0.9303,
"retention": 1,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L195",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 195,
"raw_line": " 93.03 | ",
"row_label_line": 193,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 194,
"raw_line": " 93.03 | ",
"row_label_line": 193,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 100
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7804,
"quant_score": 0.7745,
"retention": 0.9924397744746284,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L207",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 207,
"raw_line": " 77.45 | ",
"row_label_line": 205,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 206,
"raw_line": " 78.04 | ",
"row_label_line": 205,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 99.3
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.1539,
"quant_score": 0.1487,
"retention": 0.9662118258609486,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L268",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 268,
"raw_line": " 14.87 | ",
"row_label_line": 266,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 267,
"raw_line": " 15.39 | ",
"row_label_line": 266,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 96.6
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "FP8 W8A8 dynamic (FP8 weights symmetric per-channel, FP8 activations dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.4245,
"quant_score": 0.4211,
"retention": 0.9919905771495876,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/blob/fb3637a1165cec3832958bd72ebbe04021601489/README.md#L232",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 232,
"raw_line": " 42.11 | ",
"row_label_line": 230,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-FP8-dynamic/README.md",
"sha256": "cdc729dd12ecc30ebcb49481a3ea23ae26d05582a972ddb4a3da4fec4f04bec0",
"line": 231,
"raw_line": " 42.45 | ",
"row_label_line": 230,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic@fb3637a:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic",
"artifact_revision": "fb3637a1165cec3832958bd72ebbe04021601489",
"published_recovery_pct": 99.2
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.7623,
"quant_score": 0.7495,
"retention": 0.9832087104814378,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L196",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 196,
"raw_line": "74.95 | ",
"row_label_line": 194,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 195,
"raw_line": "76.23 | ",
"row_label_line": 194,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 98.23
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.6783,
"quant_score": 0.6561,
"retention": 0.9672711189739054,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L178",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 178,
"raw_line": "65.61 | ",
"row_label_line": 176,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 177,
"raw_line": "67.83 | ",
"row_label_line": 176,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 96.73
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9529,
"quant_score": 0.9519,
"retention": 0.9989505719382935,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L184",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 184,
"raw_line": "95.19 | ",
"row_label_line": 182,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 183,
"raw_line": "95.29 | ",
"row_label_line": 182,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 99.9
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.6557,
"quant_score": 0.6404,
"retention": 0.9766661583041026,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L190",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 190,
"raw_line": "64.04 | ",
"row_label_line": 188,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 189,
"raw_line": "65.57 | ",
"row_label_line": 188,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 97.67
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7574,
"quant_score": 0.7586,
"retention": 1.001584367573277,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L239",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 239,
"raw_line": " 75.86 | ",
"row_label_line": 237,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 238,
"raw_line": " 75.74 | ",
"row_label_line": 237,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 100.2
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.9303,
"quant_score": 0.9303,
"retention": 1,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L209",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 209,
"raw_line": " 93.03 | ",
"row_label_line": 207,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 208,
"raw_line": " 93.03 | ",
"row_label_line": 207,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 100
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7804,
"quant_score": 0.7715,
"retention": 0.9885955920041004,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L221",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 221,
"raw_line": " 77.15 | ",
"row_label_line": 219,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 220,
"raw_line": " 78.04 | ",
"row_label_line": 219,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 98.9
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.1539,
"quant_score": 0.1356,
"retention": 0.8810916179337231,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L282",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 282,
"raw_line": " 13.56 | ",
"row_label_line": 280,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 281,
"raw_line": " 15.39 | ",
"row_label_line": 280,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 88.1
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; card states GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.4245,
"quant_score": 0.4241,
"retention": 0.9990577149587748,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/blob/b5b2803148745cd6986a134393fea571a28af1d5/README.md#L246",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 246,
"raw_line": " 42.41 | ",
"row_label_line": 244,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w4a16/README.md",
"sha256": "50ae7b2c9b5fbb655e4806a92c63699120e7eb39952ebe961ab79c2241da43ee",
"line": 245,
"raw_line": " 42.45 | ",
"row_label_line": 244,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16@b5b2803:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16",
"artifact_revision": "b5b2803148745cd6986a134393fea571a28af1d5",
"published_recovery_pct": 99.9
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.7623,
"quant_score": 0.7602,
"retention": 0.9972451790633607,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L200",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 200,
"raw_line": "76.02 | ",
"row_label_line": 198,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 199,
"raw_line": "76.23 | ",
"row_label_line": 198,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 99.72
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "AIME 2024 pass@1",
"full": 0.6783,
"quant_score": 0.6778,
"retention": 0.9992628630399527,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L182",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 182,
"raw_line": "67.78 | ",
"row_label_line": 180,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 181,
"raw_line": "67.83 | ",
"row_label_line": 180,
"row_label": "AIME 2024 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 99.93
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MATH-500 pass@1",
"full": 0.9529,
"quant_score": 0.9527,
"retention": 0.9997901143876586,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L188",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 188,
"raw_line": "95.27 | ",
"row_label_line": 186,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 187,
"raw_line": "95.29 | ",
"row_label_line": 186,
"row_label": "MATH-500 (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 99.98
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GPQA Diamond pass@1",
"full": 0.6557,
"quant_score": 0.6501,
"retention": 0.991459508921763,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L194",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 194,
"raw_line": "65.01 | ",
"row_label_line": 192,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 193,
"raw_line": "65.57 | ",
"row_label_line": 192,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 99.15
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7574,
"quant_score": 0.7602,
"retention": 1.0036968576709797,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L243",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 243,
"raw_line": " 76.02 | ",
"row_label_line": 241,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 242,
"raw_line": " 75.74 | ",
"row_label_line": 241,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 100.4
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.9303,
"quant_score": 0.9356,
"retention": 1.0056970869611952,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L213",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 213,
"raw_line": " 93.56 | ",
"row_label_line": 211,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 212,
"raw_line": " 93.03 | ",
"row_label_line": 211,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 100.6
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7804,
"quant_score": 0.7801,
"retention": 0.9996155817529471,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L225",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 225,
"raw_line": " 78.01 | ",
"row_label_line": 223,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 224,
"raw_line": " 78.04 | ",
"row_label_line": 223,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 100
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.1539,
"quant_score": 0.1523,
"retention": 0.9896036387264456,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L286",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 286,
"raw_line": " 15.23 | ",
"row_label_line": 284,
"row_label": "Average Score",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 285,
"raw_line": " 15.39 | ",
"row_label_line": 284,
"row_label": "Average Score",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 99
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "INT8 W8A8 (weights symmetric per-channel, activations symmetric per-token; SmoothQuant + GPTQ via llm-compressor)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config, chat template)",
"full": 0.4245,
"quant_score": 0.4251,
"retention": 1.0014134275618374,
"baseline_quant": "BF16 (unquantized deepseek-ai/DeepSeek-R1-Distill-Llama-70B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/blob/2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf/README.md#L250",
"source_id": "quant-redhatai-deepseek-r1-distill-llama-70b-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 250,
"raw_line": " 42.51 | ",
"row_label_line": 248,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-Distill-Llama-70B-quantized.w8a8/README.md",
"sha256": "2700ed8b3280c16c29c80e6bc58c232d345120390a83703dc167783fe107cb28",
"line": 249,
"raw_line": " 42.45 | ",
"row_label_line": 248,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Neural Magic (now Red Hat) evaluation with vLLM; OpenLLM tasks via lm-evaluation-harness per the card commands; reasoning-task harness/sampling not stated in this card. Baseline values differ from DeepSeek's own reported scores, i.e. re-measured. Weights derive from Llama 3.x; the repository MIT declaration is relied on only for the card text/data, not as a statement about weight licensing. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8@2fb57dc:openllm-v2",
"artifact": "RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8",
"artifact_revision": "2fb57dc0dc61ccf5b282f96e9a6b939b447ecdbf",
"published_recovery_pct": 100.1
},
{
"model": "DeepSeek-R1",
"model_id": "deepseek-r1",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.8104,
"quant_score": 0.8128,
"retention": 1.0029615004935835,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-quantized.w4a16/blob/12a44c05b1b94f9759420d9acc05328921f76dde/README.md#L111",
"source_id": "quant-redhatai-deepseek-r1-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 111,
"raw_line": "| **OpenLLM v1
Average Score** | **100.30** | **81.04** | **81.28** |",
"row_label_line": 111,
"row_label": "OpenLLM v1 Average Score",
"column_header": "RedHatAI/DeepSeek-R1-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 111,
"raw_line": "| **OpenLLM v1
Average Score** | **100.30** | **81.04** | **81.28** |",
"row_label_line": 111,
"row_label": "OpenLLM v1 Average Score",
"column_header": "deepseek/DeepSeek-R1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16; it is the full-precision release of this model. Reasoning: LightEval (open-r1), pass@1 estimated from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 32768; OpenLLM v1 via lm-evaluation-harness; vLLM tensor parallel 8. Baseline scores differ from DeepSeek-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/DeepSeek-R1-quantized.w4a16@12a44c0:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-quantized.w4a16",
"artifact_revision": "12a44c05b1b94f9759420d9acc05328921f76dde",
"published_recovery_pct": 100.3
},
{
"model": "DeepSeek-R1",
"model_id": "deepseek-r1",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "GSM8K 5-shot (lm-evaluation-harness, vLLM)",
"full": 0.9591,
"quant_score": 0.9568,
"retention": 0.9976019184652279,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-quantized.w4a16/blob/12a44c05b1b94f9759420d9acc05328921f76dde/README.md#L106",
"source_id": "quant-redhatai-deepseek-r1-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 106,
"raw_line": "| GSM8k
5-shot | 99.76 | 95.91 | 95.68 |",
"row_label_line": 106,
"row_label": "GSM8k 5-shot",
"column_header": "RedHatAI/DeepSeek-R1-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 106,
"raw_line": "| GSM8k
5-shot | 99.76 | 95.91 | 95.68 |",
"row_label_line": 106,
"row_label": "GSM8k 5-shot",
"column_header": "deepseek/DeepSeek-R1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16; it is the full-precision release of this model. Reasoning: LightEval (open-r1), pass@1 estimated from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 32768; OpenLLM v1 via lm-evaluation-harness; vLLM tensor parallel 8. Baseline scores differ from DeepSeek-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-quantized.w4a16@12a44c0:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-quantized.w4a16",
"artifact_revision": "12a44c05b1b94f9759420d9acc05328921f76dde",
"published_recovery_pct": 99.76
},
{
"model": "DeepSeek-R1",
"model_id": "deepseek-r1",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "MMLU 5-shot (lm-evaluation-harness, vLLM)",
"full": 0.8722,
"quant_score": 0.8699,
"retention": 0.9973629901398761,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-quantized.w4a16/blob/12a44c05b1b94f9759420d9acc05328921f76dde/README.md#L108",
"source_id": "quant-redhatai-deepseek-r1-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 108,
"raw_line": "| MMLU
5-shot | 99.74 | 87.22 | 86.99 |",
"row_label_line": 108,
"row_label": "MMLU 5-shot",
"column_header": "RedHatAI/DeepSeek-R1-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 108,
"raw_line": "| MMLU
5-shot | 99.74 | 87.22 | 86.99 |",
"row_label_line": 108,
"row_label": "MMLU 5-shot",
"column_header": "deepseek/DeepSeek-R1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16; it is the full-precision release of this model. Reasoning: LightEval (open-r1), pass@1 estimated from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 32768; OpenLLM v1 via lm-evaluation-harness; vLLM tensor parallel 8. Baseline scores differ from DeepSeek-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-quantized.w4a16@12a44c0:openllm-v1",
"artifact": "RedHatAI/DeepSeek-R1-quantized.w4a16",
"artifact_revision": "12a44c05b1b94f9759420d9acc05328921f76dde",
"published_recovery_pct": 99.74
},
{
"model": "DeepSeek-R1",
"model_id": "deepseek-r1",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.8299,
"quant_score": 0.82,
"retention": 0.9880708519098687,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-quantized.w4a16/blob/12a44c05b1b94f9759420d9acc05328921f76dde/README.md#L115",
"source_id": "quant-redhatai-deepseek-r1-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 115,
"raw_line": "| **Reasoning
Average Score** | **98.81** | **82.99** | **82.00** |",
"row_label_line": 115,
"row_label": "Reasoning Average Score",
"column_header": "RedHatAI/DeepSeek-R1-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 115,
"raw_line": "| **Reasoning
Average Score** | **98.81** | **82.99** | **82.00** |",
"row_label_line": 115,
"row_label": "Reasoning Average Score",
"column_header": "deepseek/DeepSeek-R1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16; it is the full-precision release of this model. Reasoning: LightEval (open-r1), pass@1 estimated from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 32768; OpenLLM v1 via lm-evaluation-harness; vLLM tensor parallel 8. Baseline scores differ from DeepSeek-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/DeepSeek-R1-quantized.w4a16@12a44c0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-quantized.w4a16",
"artifact_revision": "12a44c05b1b94f9759420d9acc05328921f76dde",
"published_recovery_pct": 98.81
},
{
"model": "DeepSeek-R1",
"model_id": "deepseek-r1",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "AIME 2024 pass@1 (LightEval/open-r1, mean of 10 seeds)",
"full": 0.7833,
"quant_score": 0.77,
"retention": 0.9830205540661305,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-quantized.w4a16/blob/12a44c05b1b94f9759420d9acc05328921f76dde/README.md#L112",
"source_id": "quant-redhatai-deepseek-r1-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 112,
"raw_line": "| AIME 2024
pass@1 | 98.30 | 78.33 | 77.00 |",
"row_label_line": 112,
"row_label": "AIME 2024 pass@1",
"column_header": "RedHatAI/DeepSeek-R1-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 112,
"raw_line": "| AIME 2024
pass@1 | 98.30 | 78.33 | 77.00 |",
"row_label_line": 112,
"row_label": "AIME 2024 pass@1",
"column_header": "deepseek/DeepSeek-R1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16; it is the full-precision release of this model. Reasoning: LightEval (open-r1), pass@1 estimated from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 32768; OpenLLM v1 via lm-evaluation-harness; vLLM tensor parallel 8. Baseline scores differ from DeepSeek-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-quantized.w4a16@12a44c0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-quantized.w4a16",
"artifact_revision": "12a44c05b1b94f9759420d9acc05328921f76dde",
"published_recovery_pct": 98.3
},
{
"model": "DeepSeek-R1",
"model_id": "deepseek-r1",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "MATH-500 pass@1 (LightEval/open-r1, mean of 10 seeds)",
"full": 0.9724,
"quant_score": 0.9708,
"retention": 0.9983545865898807,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-quantized.w4a16/blob/12a44c05b1b94f9759420d9acc05328921f76dde/README.md#L113",
"source_id": "quant-redhatai-deepseek-r1-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 113,
"raw_line": "| MATH-500
pass@1 | 99.84 | 97.24 | 97.08 |",
"row_label_line": 113,
"row_label": "MATH-500 pass@1",
"column_header": "RedHatAI/DeepSeek-R1-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 113,
"raw_line": "| MATH-500
pass@1 | 99.84 | 97.24 | 97.08 |",
"row_label_line": 113,
"row_label": "MATH-500 pass@1",
"column_header": "deepseek/DeepSeek-R1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16; it is the full-precision release of this model. Reasoning: LightEval (open-r1), pass@1 estimated from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 32768; OpenLLM v1 via lm-evaluation-harness; vLLM tensor parallel 8. Baseline scores differ from DeepSeek-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-quantized.w4a16@12a44c0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-quantized.w4a16",
"artifact_revision": "12a44c05b1b94f9759420d9acc05328921f76dde",
"published_recovery_pct": 99.84
},
{
"model": "DeepSeek-R1",
"model_id": "deepseek-r1",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "GPQA Diamond pass@1 (LightEval/open-r1, mean of 10 seeds)",
"full": 0.7338,
"quant_score": 0.7192,
"retention": 0.9801035704551649,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-quantized.w4a16/blob/12a44c05b1b94f9759420d9acc05328921f76dde/README.md#L114",
"source_id": "quant-redhatai-deepseek-r1-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 114,
"raw_line": "| GPQA Diamond
pass@1 | 98.01 | 73.38 | 71.92 |",
"row_label_line": 114,
"row_label": "GPQA Diamond pass@1",
"column_header": "RedHatAI/DeepSeek-R1-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-quantized.w4a16/README.md",
"sha256": "ed56f4907022de0f866ed66cde92b96db545eafab1f8587c72740c234f13c1c0",
"line": 114,
"raw_line": "| GPQA Diamond
pass@1 | 98.01 | 73.38 | 71.92 |",
"row_label_line": 114,
"row_label": "GPQA Diamond pass@1",
"column_header": "deepseek/DeepSeek-R1"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16; it is the full-precision release of this model. Reasoning: LightEval (open-r1), pass@1 estimated from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 32768; OpenLLM v1 via lm-evaluation-harness; vLLM tensor parallel 8. Baseline scores differ from DeepSeek-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-quantized.w4a16@12a44c0:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-quantized.w4a16",
"artifact_revision": "12a44c05b1b94f9759420d9acc05328921f76dde",
"published_recovery_pct": 98.01
},
{
"model": "DeepSeek-R1-0528",
"model_id": "deepseek-r1-0528",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "AIME 2024 pass@1 (LightEval/open-r1, mean of 10 seeds)",
"full": 0.8866,
"quant_score": 0.8733,
"retention": 0.9849988720956463,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1-0528 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-0528-quantized.w4a16/blob/0efe34e82e4612e726c42f6fd44116e0244b33f8/README.md#L238",
"source_id": "quant-redhatai-deepseek-r1-0528-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-0528-quantized.w4a16/README.md",
"sha256": "6926e4160e68e3de7bf3631a6e1f5d5c30f2d753beeaf9bbee74d28e69f78dd3",
"line": 238,
"raw_line": "| AIME 2024
pass@1 | 98.50 | 88.66 | 87.33 |",
"row_label_line": 238,
"row_label": "AIME 2024 pass@1",
"column_header": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-0528-quantized.w4a16/README.md",
"sha256": "6926e4160e68e3de7bf3631a6e1f5d5c30f2d753beeaf9bbee74d28e69f78dd3",
"line": 238,
"raw_line": "| AIME 2024
pass@1 | 98.50 | 88.66 | 87.33 |",
"row_label_line": 238,
"row_label": "AIME 2024 pass@1",
"column_header": "deepseek/DeepSeek-R1-0528"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16. LightEval (open-r1) pass@1 from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 65536. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16@0efe34e:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16",
"artifact_revision": "0efe34e82e4612e726c42f6fd44116e0244b33f8",
"published_recovery_pct": 98.5
},
{
"model": "DeepSeek-R1-0528",
"model_id": "deepseek-r1-0528",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "MATH-500 pass@1 (LightEval/open-r1, mean of 10 seeds)",
"full": 0.9752,
"quant_score": 0.974,
"retention": 0.998769483182937,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1-0528 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-0528-quantized.w4a16/blob/0efe34e82e4612e726c42f6fd44116e0244b33f8/README.md#L239",
"source_id": "quant-redhatai-deepseek-r1-0528-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-0528-quantized.w4a16/README.md",
"sha256": "6926e4160e68e3de7bf3631a6e1f5d5c30f2d753beeaf9bbee74d28e69f78dd3",
"line": 239,
"raw_line": "| MATH-500
pass@1 | 99.88 | 97.52 | 97.40 |",
"row_label_line": 239,
"row_label": "MATH-500 pass@1",
"column_header": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-0528-quantized.w4a16/README.md",
"sha256": "6926e4160e68e3de7bf3631a6e1f5d5c30f2d753beeaf9bbee74d28e69f78dd3",
"line": 239,
"raw_line": "| MATH-500
pass@1 | 99.88 | 97.52 | 97.40 |",
"row_label_line": 239,
"row_label": "MATH-500 pass@1",
"column_header": "deepseek/DeepSeek-R1-0528"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16. LightEval (open-r1) pass@1 from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 65536. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16@0efe34e:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16",
"artifact_revision": "0efe34e82e4612e726c42f6fd44116e0244b33f8",
"published_recovery_pct": 99.88
},
{
"model": "DeepSeek-R1-0528",
"model_id": "deepseek-r1-0528",
"params_b": 685,
"quant": "INT4 W4A16 (weights only; GPTQ via MoE-Quant, Red Hat/ISTA)",
"metric": "GPQA Diamond pass@1 (LightEval/open-r1, mean of 10 seeds)",
"full": 0.7965,
"quant_score": 0.8061,
"retention": 1.0120527306967984,
"baseline_quant": "FP8 (original deepseek-ai/DeepSeek-R1-0528 release weights = native training precision; evaluated by Red Hat in the same study)",
"source": "https://huggingface.co/RedHatAI/DeepSeek-R1-0528-quantized.w4a16/blob/0efe34e82e4612e726c42f6fd44116e0244b33f8/README.md#L240",
"source_id": "quant-redhatai-deepseek-r1-0528-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-0528-quantized.w4a16/README.md",
"sha256": "6926e4160e68e3de7bf3631a6e1f5d5c30f2d753beeaf9bbee74d28e69f78dd3",
"line": 240,
"raw_line": "| GPQA Diamond
pass@1 | 101.21 | 79.65 | 80.61 |",
"row_label_line": 240,
"row_label": "GPQA Diamond pass@1",
"column_header": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__DeepSeek-R1-0528-quantized.w4a16/README.md",
"sha256": "6926e4160e68e3de7bf3631a6e1f5d5c30f2d753beeaf9bbee74d28e69f78dd3",
"line": 240,
"raw_line": "| GPQA Diamond
pass@1 | 101.21 | 79.65 | 80.61 |",
"row_label_line": 240,
"row_label": "GPQA Diamond pass@1",
"column_header": "deepseek/DeepSeek-R1-0528"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline is the published native-FP8 checkpoint, not BF16. LightEval (open-r1) pass@1 from 10 seeds, temperature 0.6, top_p 0.95, max_new_tokens 65536. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16@0efe34e:reasoning",
"artifact": "RedHatAI/DeepSeek-R1-0528-quantized.w4a16",
"artifact_revision": "0efe34e82e4612e726c42f6fd44116e0244b33f8",
"published_recovery_pct": 101.21
},
{
"model": "Devstral-Small-2507",
"model_id": null,
"params_b": null,
"quant": "FP8 W8A8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "EvalPlus average of 4 tasks (HumanEval, HumanEval+, MBPP, MBPP+; greedy pass@1; vLLM 0.10.1.1)",
"full": 0.7843,
"quant_score": 0.7818,
"retention": 0.9968124442177737,
"baseline_quant": "BF16 (unquantized mistralai/Devstral-Small-2507, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Devstral-Small-2507-FP8-Dynamic/blob/0566e03128c6190f34931e750653c2f5334174d7/README.md#L94",
"source_id": "quant-redhatai-devstral-small-2507-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Devstral-Small-2507-FP8-Dynamic/README.md",
"sha256": "aed6e4dfa0cfe4dd40e06041880da5a8ad040d13e314a1d7f670e5eeac94f9bf",
"line": 94,
"raw_line": "| **Average Score** | **99.68** | **78.43** | **78.18** |",
"row_label_line": 94,
"row_label": "Average Score",
"column_header": "RedHatAI/Devstral-Small-2507-FP8-Dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Devstral-Small-2507-FP8-Dynamic/README.md",
"sha256": "aed6e4dfa0cfe4dd40e06041880da5a8ad040d13e314a1d7f670e5eeac94f9bf",
"line": 94,
"raw_line": "| **Average Score** | **99.68** | **78.43** | **78.18** |",
"row_label_line": 94,
"row_label": "Average Score",
"column_header": "mistralai/Devstral-Small-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Devstral-Small-2507 is not the Devstral Small 2 (2512) checkpoint in data/models.json; model_id deliberately null. params_b not stated in card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 4 tasks: EvalPlus",
"study": "RedHatAI/Devstral-Small-2507-FP8-Dynamic@0566e03:evalplus",
"artifact": "RedHatAI/Devstral-Small-2507-FP8-Dynamic",
"artifact_revision": "0566e03128c6190f34931e750653c2f5334174d7",
"published_recovery_pct": 99.68
},
{
"model": "Devstral-Small-2507",
"model_id": null,
"params_b": null,
"quant": "INT4 W4A16 (GPTQ, group size 128 per creation code; card overview lists weight/activation fields swapped)",
"metric": "EvalPlus average of 4 tasks (HumanEval, HumanEval+, MBPP, MBPP+; greedy pass@1; vLLM 0.10.1.1)",
"full": 0.7843,
"quant_score": 0.7828,
"retention": 0.9980874665306643,
"baseline_quant": "BF16 (unquantized mistralai/Devstral-Small-2507, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Devstral-Small-2507-quantized.w4a16/blob/bb96f0d1a876d6f6f3ed657b6a965328697ba2f9/README.md#L200",
"source_id": "quant-redhatai-devstral-small-2507-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__Devstral-Small-2507-quantized.w4a16/README.md",
"sha256": "e2e11d911eabf357bdcc0921cebcd8a99ea68f172942d24fe133a9f08027a35b",
"line": 200,
"raw_line": "| **Average Score** | **99.81** | **78.43** | **78.28** |",
"row_label_line": 200,
"row_label": "Average Score",
"column_header": "RedHatAI/Devstral-Small-2507-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Devstral-Small-2507-quantized.w4a16/README.md",
"sha256": "e2e11d911eabf357bdcc0921cebcd8a99ea68f172942d24fe133a9f08027a35b",
"line": 200,
"raw_line": "| **Average Score** | **99.81** | **78.43** | **78.28** |",
"row_label_line": 200,
"row_label": "Average Score",
"column_header": "mistralai/Devstral-Small-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Devstral-Small-2507 is not the Devstral Small 2 (2512) checkpoint in data/models.json; model_id deliberately null. params_b not stated in card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 4 tasks: EvalPlus",
"study": "RedHatAI/Devstral-Small-2507-quantized.w4a16@bb96f0d:evalplus",
"artifact": "RedHatAI/Devstral-Small-2507-quantized.w4a16",
"artifact_revision": "bb96f0d1a876d6f6f3ed657b6a965328697ba2f9",
"published_recovery_pct": 99.81
},
{
"model": "Devstral-Small-2507",
"model_id": null,
"params_b": null,
"quant": "INT8 W8A8 (SmoothQuant + GPTQ via llm-compressor)",
"metric": "EvalPlus average of 4 tasks (HumanEval, HumanEval+, MBPP, MBPP+; greedy pass@1; vLLM 0.10.1.1)",
"full": 0.7843,
"quant_score": 0.7903,
"retention": 1.0076501338773427,
"baseline_quant": "BF16 (unquantized mistralai/Devstral-Small-2507, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Devstral-Small-2507-quantized.w8a8/blob/b152b0574658cdd159721472ebf5353d8d199451/README.md#L164",
"source_id": "quant-redhatai-devstral-small-2507-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__Devstral-Small-2507-quantized.w8a8/README.md",
"sha256": "053242b1b336d77b76919c098e2c65c8f2a546eb1a07f7109554fea083adf426",
"line": 164,
"raw_line": "| **Average Score** | **100.77** | **78.43** | **79.03** |",
"row_label_line": 164,
"row_label": "Average Score",
"column_header": "RedHatAI/Devstral-Small-2507-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Devstral-Small-2507-quantized.w8a8/README.md",
"sha256": "053242b1b336d77b76919c098e2c65c8f2a546eb1a07f7109554fea083adf426",
"line": 164,
"raw_line": "| **Average Score** | **100.77** | **78.43** | **79.03** |",
"row_label_line": 164,
"row_label": "Average Score",
"column_header": "mistralai/Devstral-Small-2507"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Devstral-Small-2507 is not the Devstral Small 2 (2512) checkpoint in data/models.json; model_id deliberately null. params_b not stated in card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 4 tasks: EvalPlus",
"study": "RedHatAI/Devstral-Small-2507-quantized.w8a8@b152b05:evalplus",
"artifact": "RedHatAI/Devstral-Small-2507-quantized.w8a8",
"artifact_revision": "b152b0574658cdd159721472ebf5353d8d199451",
"published_recovery_pct": 100.77
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "FP8 W8A8 dynamic (weights per-channel, activations per-token; llm-compressor)",
"metric": "Reasoning average of 3 tasks (AIME 2025, GPQA Diamond, and a third task labelled \"Math-lvl-5\" in the table but run with lighteval `math_500` per the card commands)",
"full": 0.6129,
"quant_score": 0.6185,
"retention": 1.009136890194159,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic/blob/c5d7753e839c659e04906bcfce7eb6ea50d6d18c/README.md#L265",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 265,
"raw_line": " 61.85",
"row_label_line": 261,
"row_label": "Average",
"column_header": "Ministral-3-14B-Instruct-2512-FP8-dynamic (this model)",
"category": "Reasoning (generation)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 263,
"raw_line": " | 61.29",
"row_label_line": 261,
"row_label": "Average",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic@c5d7753:reasoning",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic",
"artifact_revision": "c5d7753e839c659e04906bcfce7eb6ea50d6d18c",
"published_recovery_pct": 100.9
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "FP8 W8A8 dynamic (weights per-channel, activations per-token; llm-compressor)",
"metric": "AIME 2025 (lighteval, vLLM, temperature 0.15)",
"full": 0.3667,
"quant_score": 0.3,
"retention": 0.8181074447777474,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic/blob/c5d7753e839c659e04906bcfce7eb6ea50d6d18c/README.md#L235",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 235,
"raw_line": " | 30.0",
"row_label_line": 231,
"row_label": "AIME 2025",
"column_header": "Ministral-3-14B-Instruct-2512-FP8-dynamic (this model)",
"category": "Reasoning (generation)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 233,
"raw_line": " | 36.67",
"row_label_line": 231,
"row_label": "AIME 2025",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic@c5d7753:reasoning",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic",
"artifact_revision": "c5d7753e839c659e04906bcfce7eb6ea50d6d18c",
"published_recovery_pct": 81.81
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "FP8 W8A8 dynamic (weights per-channel, activations per-token; llm-compressor)",
"metric": "GPQA Diamond (lighteval, vLLM, temperature 0.15)",
"full": 0.5859,
"quant_score": 0.6616,
"retention": 1.1292029356545483,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic/blob/c5d7753e839c659e04906bcfce7eb6ea50d6d18c/README.md#L245",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 245,
"raw_line": " | 66.16",
"row_label_line": 241,
"row_label": "GPQA diamond",
"column_header": "Ministral-3-14B-Instruct-2512-FP8-dynamic (this model)",
"category": "Reasoning (generation)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 243,
"raw_line": " | 58.59",
"row_label_line": 241,
"row_label": "GPQA diamond",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic@c5d7753:reasoning",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic",
"artifact_revision": "c5d7753e839c659e04906bcfce7eb6ea50d6d18c",
"published_recovery_pct": 112.9
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "FP8 W8A8 dynamic (weights per-channel, activations per-token; llm-compressor)",
"metric": "IFEval 0-shot (lm-evaluation-harness, chat template, vLLM)",
"full": 0.7734,
"quant_score": 0.7686,
"retention": 0.9937936384794412,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic/blob/c5d7753e839c659e04906bcfce7eb6ea50d6d18c/README.md#L223",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 223,
"raw_line": " | 76.86",
"row_label_line": 219,
"row_label": "IFEval (0-shot)",
"column_header": "Ministral-3-14B-Instruct-2512-FP8-dynamic (this model)",
"category": "OpenLLM v2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-FP8-dynamic/README.md",
"sha256": "ce9c75e033dd09c05ab0176ec021fcf510444ba86804f50b4367ef189c71e5c6",
"line": 221,
"raw_line": " | 77.34",
"row_label_line": 219,
"row_label": "IFEval (0-shot)",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic@c5d7753:ifeval",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic",
"artifact_revision": "c5d7753e839c659e04906bcfce7eb6ea50d6d18c",
"published_recovery_pct": 99.4
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "NVFP4 (FP4 weights and activations, llm-compressor NVFP4 scheme)",
"metric": "Reasoning average of 3 tasks (AIME 2025, GPQA Diamond, and a third task labelled \"Math-lvl-5\" in the table but run with lighteval `math_500` per the card commands)",
"full": 0.6129,
"quant_score": 0.5975,
"retention": 0.9748735519660631,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4/blob/ae31e2b3d5aca78b8721e071291d7feed5a87dad/README.md#L250",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 250,
"raw_line": " | 59.75",
"row_label_line": 246,
"row_label": "Average",
"column_header": "Ministral-3-14B-Instruct-2512-NVFP4 (this model)",
"category": "Reasoning (generation)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 248,
"raw_line": " | 61.29",
"row_label_line": 246,
"row_label": "Average",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4@ae31e2b:reasoning",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4",
"artifact_revision": "ae31e2b3d5aca78b8721e071291d7feed5a87dad",
"published_recovery_pct": 97.49
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "NVFP4 (FP4 weights and activations, llm-compressor NVFP4 scheme)",
"metric": "AIME 2025 (lighteval, vLLM, temperature 0.15)",
"full": 0.3667,
"quant_score": 0.325,
"retention": 0.8862830651758931,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4/blob/ae31e2b3d5aca78b8721e071291d7feed5a87dad/README.md#L220",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 220,
"raw_line": " | 32.5",
"row_label_line": 216,
"row_label": "AIME 2025",
"column_header": "Ministral-3-14B-Instruct-2512-NVFP4 (this model)",
"category": "Reasoning (generation)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 218,
"raw_line": " | 36.67",
"row_label_line": 216,
"row_label": "AIME 2025",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4@ae31e2b:reasoning",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4",
"artifact_revision": "ae31e2b3d5aca78b8721e071291d7feed5a87dad",
"published_recovery_pct": 88.63
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "NVFP4 (FP4 weights and activations, llm-compressor NVFP4 scheme)",
"metric": "GPQA Diamond (lighteval, vLLM, temperature 0.15)",
"full": 0.5859,
"quant_score": 0.6094,
"retention": 1.0401092336576205,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4/blob/ae31e2b3d5aca78b8721e071291d7feed5a87dad/README.md#L230",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 230,
"raw_line": " | 60.94",
"row_label_line": 226,
"row_label": "GPQA diamond",
"column_header": "Ministral-3-14B-Instruct-2512-NVFP4 (this model)",
"category": "Reasoning (generation)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 228,
"raw_line": " | 58.59",
"row_label_line": 226,
"row_label": "GPQA diamond",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4@ae31e2b:reasoning",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4",
"artifact_revision": "ae31e2b3d5aca78b8721e071291d7feed5a87dad",
"published_recovery_pct": 104.02
},
{
"model": "Ministral-3-14B-Instruct-2512",
"model_id": "ministral-3-14b",
"params_b": 13.9,
"quant": "NVFP4 (FP4 weights and activations, llm-compressor NVFP4 scheme)",
"metric": "IFEval 0-shot (lm-evaluation-harness, chat template, vLLM)",
"full": 0.7734,
"quant_score": 0.6355,
"retention": 0.8216964054822858,
"baseline_quant": "BF16 (mistralai/Ministral-3-14B-Instruct-2512-BF16, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4/blob/ae31e2b3d5aca78b8721e071291d7feed5a87dad/README.md#L208",
"source_id": "quant-redhatai-ministral-3-14b-instruct-2512-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 208,
"raw_line": " | 63.55",
"row_label_line": 204,
"row_label": "IFEval",
"column_header": "Ministral-3-14B-Instruct-2512-NVFP4 (this model)",
"category": "OpenLLM v2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Ministral-3-14B-Instruct-2512-NVFP4/README.md",
"sha256": "7c08e42d223ad3fb3f1f130da67cace3e153830b880c331ceacebf712d552250",
"line": 206,
"raw_line": " | 77.34",
"row_label_line": 204,
"row_label": "IFEval",
"column_header": "Ministral-3-14B-Instruct-2512-BF16"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. The single MATH row is not recorded because its label (\"Math-lvl-5\") conflicts with the command shown (math_500). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4@ae31e2b:ifeval",
"artifact": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4",
"artifact_revision": "ae31e2b3d5aca78b8721e071291d7feed5a87dad",
"published_recovery_pct": 82.17
},
{
"model": "Mistral-Small-3.2-24B-Instruct-2506",
"model_id": "mistral-small-3-2",
"params_b": 24,
"quant": "NVFP4 (FP4 weights and activations, tensor-group size 16; SmoothQuant + QuantizationModifier via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7825,
"quant_score": 0.7676,
"retention": 0.9809584664536742,
"baseline_quant": "BF16 (unsloth/Mistral-Small-3.2-24B-Instruct-2506 transformers-format upload of the same checkpoint, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4/blob/5102662a785ad4f2c47eb54624bf3132ee620a58/README.md#L255",
"source_id": "quant-redhatai-mistral-small-3-2-24b-instruct-2506-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 255,
"raw_line": " | 76.76 | ",
"row_label_line": 253,
"row_label": "Average",
"column_header": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 254,
"raw_line": " 78.25 | ",
"row_label_line": 253,
"row_label": "Average",
"column_header": "unsloth/Mistral-Small-3.2-24B-Instruct-2506"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Evaluated with the neuralmagic fork of lm-evaluation-harness. Baseline is Unsloth's transformers-format upload that the quantization started from, not mistralai's original files. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4@5102662:openllm-v1",
"artifact": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"artifact_revision": "5102662a785ad4f2c47eb54624bf3132ee620a58",
"published_recovery_pct": 98.1
},
{
"model": "Mistral-Small-3.2-24B-Instruct-2506",
"model_id": "mistral-small-3-2",
"params_b": 24,
"quant": "NVFP4 (FP4 weights and activations, tensor-group size 16; SmoothQuant + QuantizationModifier via llm-compressor)",
"metric": "GSM8K (OpenLLM v1 config, lm-evaluation-harness)",
"full": 0.8961,
"quant_score": 0.8711,
"retention": 0.9721013279767883,
"baseline_quant": "BF16 (unsloth/Mistral-Small-3.2-24B-Instruct-2506 transformers-format upload of the same checkpoint, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4/blob/5102662a785ad4f2c47eb54624bf3132ee620a58/README.md#L225",
"source_id": "quant-redhatai-mistral-small-3-2-24b-instruct-2506-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 225,
"raw_line": " 87.11 | ",
"row_label_line": 223,
"row_label": "gsm8k",
"column_header": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 224,
"raw_line": " 89.61 | ",
"row_label_line": 223,
"row_label": "gsm8k",
"column_header": "unsloth/Mistral-Small-3.2-24B-Instruct-2506"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Evaluated with the neuralmagic fork of lm-evaluation-harness. Baseline is Unsloth's transformers-format upload that the quantization started from, not mistralai's original files. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4@5102662:openllm-v1",
"artifact": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"artifact_revision": "5102662a785ad4f2c47eb54624bf3132ee620a58",
"published_recovery_pct": 97.21
},
{
"model": "Mistral-Small-3.2-24B-Instruct-2506",
"model_id": "mistral-small-3-2",
"params_b": 24,
"quant": "NVFP4 (FP4 weights and activations, tensor-group size 16; SmoothQuant + QuantizationModifier via llm-compressor)",
"metric": "MMLU (OpenLLM v1 config, lm-evaluation-harness)",
"full": 0.8106,
"quant_score": 0.7943,
"retention": 0.9798914384406614,
"baseline_quant": "BF16 (unsloth/Mistral-Small-3.2-24B-Instruct-2506 transformers-format upload of the same checkpoint, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4/blob/5102662a785ad4f2c47eb54624bf3132ee620a58/README.md#L237",
"source_id": "quant-redhatai-mistral-small-3-2-24b-instruct-2506-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 237,
"raw_line": " 79.43 | ",
"row_label_line": 235,
"row_label": "mmlu",
"column_header": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 236,
"raw_line": " 81.06 | ",
"row_label_line": 235,
"row_label": "mmlu",
"column_header": "unsloth/Mistral-Small-3.2-24B-Instruct-2506"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Evaluated with the neuralmagic fork of lm-evaluation-harness. Baseline is Unsloth's transformers-format upload that the quantization started from, not mistralai's original files. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4@5102662:openllm-v1",
"artifact": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"artifact_revision": "5102662a785ad4f2c47eb54624bf3132ee620a58",
"published_recovery_pct": 97.99
},
{
"model": "Mistral-Small-3.2-24B-Instruct-2506",
"model_id": "mistral-small-3-2",
"params_b": 24,
"quant": "NVFP4 (FP4 weights and activations, tensor-group size 16; SmoothQuant + QuantizationModifier via llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.5246,
"quant_score": 0.5124,
"retention": 0.9767441860465117,
"baseline_quant": "BF16 (unsloth/Mistral-Small-3.2-24B-Instruct-2506 transformers-format upload of the same checkpoint, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4/blob/5102662a785ad4f2c47eb54624bf3132ee620a58/README.md#L298",
"source_id": "quant-redhatai-mistral-small-3-2-24b-instruct-2506-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 298,
"raw_line": " 51.24 | ",
"row_label_line": 296,
"row_label": "Average",
"column_header": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 297,
"raw_line": " 52.46 | ",
"row_label_line": 296,
"row_label": "Average",
"column_header": "unsloth/Mistral-Small-3.2-24B-Instruct-2506"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Evaluated with the neuralmagic fork of lm-evaluation-harness. Baseline is Unsloth's transformers-format upload that the quantization started from, not mistralai's original files. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4@5102662:openllm-v2",
"artifact": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"artifact_revision": "5102662a785ad4f2c47eb54624bf3132ee620a58",
"published_recovery_pct": 97.68
},
{
"model": "Mistral-Small-3.2-24B-Instruct-2506",
"model_id": "mistral-small-3-2",
"params_b": 24,
"quant": "NVFP4 (FP4 weights and activations, tensor-group size 16; SmoothQuant + QuantizationModifier via llm-compressor)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config)",
"full": 0.5084,
"quant_score": 0.4845,
"retention": 0.9529897718332021,
"baseline_quant": "BF16 (unsloth/Mistral-Small-3.2-24B-Instruct-2506 transformers-format upload of the same checkpoint, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4/blob/5102662a785ad4f2c47eb54624bf3132ee620a58/README.md#L268",
"source_id": "quant-redhatai-mistral-small-3-2-24b-instruct-2506-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 268,
"raw_line": " 48.45 | ",
"row_label_line": 266,
"row_label": "MMLU-Pro (5-shot)",
"column_header": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 267,
"raw_line": " 50.84 | ",
"row_label_line": 266,
"row_label": "MMLU-Pro (5-shot)",
"column_header": "unsloth/Mistral-Small-3.2-24B-Instruct-2506"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Evaluated with the neuralmagic fork of lm-evaluation-harness. Baseline is Unsloth's transformers-format upload that the quantization started from, not mistralai's original files. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4@5102662:openllm-v2",
"artifact": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"artifact_revision": "5102662a785ad4f2c47eb54624bf3132ee620a58",
"published_recovery_pct": 95.3
},
{
"model": "Mistral-Small-3.2-24B-Instruct-2506",
"model_id": "mistral-small-3-2",
"params_b": 24,
"quant": "NVFP4 (FP4 weights and activations, tensor-group size 16; SmoothQuant + QuantizationModifier via llm-compressor)",
"metric": "IFEval 0-shot (OpenLLM v2 config)",
"full": 0.8405,
"quant_score": 0.8441,
"retention": 1.0042831647828672,
"baseline_quant": "BF16 (unsloth/Mistral-Small-3.2-24B-Instruct-2506 transformers-format upload of the same checkpoint, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4/blob/5102662a785ad4f2c47eb54624bf3132ee620a58/README.md#L280",
"source_id": "quant-redhatai-mistral-small-3-2-24b-instruct-2506-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 280,
"raw_line": " 84.41 | ",
"row_label_line": 278,
"row_label": "IFEval (0-shot)",
"column_header": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Mistral-Small-3.2-24B-Instruct-2506-NVFP4/README.md",
"sha256": "d4d012658448ff64124a73de203328c03574e27e596e349106843ceff2f81ae3",
"line": 279,
"raw_line": " 84.05 | ",
"row_label_line": 278,
"row_label": "IFEval (0-shot)",
"column_header": "unsloth/Mistral-Small-3.2-24B-Instruct-2506"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Evaluated with the neuralmagic fork of lm-evaluation-harness. Baseline is Unsloth's transformers-format upload that the quantization started from, not mistralai's original files. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4@5102662:openllm-v2",
"artifact": "RedHatAI/Mistral-Small-3.2-24B-Instruct-2506-NVFP4",
"artifact_revision": "5102662a785ad4f2c47eb54624bf3132ee620a58",
"published_recovery_pct": 100.43
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7652,
"quant_score": 0.7647,
"retention": 0.9993465760585467,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-FP8-dynamic/blob/8f62f4b85ccc1c7fda9f178976c1162f37e43651/README.md#L397",
"source_id": "quant-redhatai-phi-4-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-FP8-dynamic/README.md",
"sha256": "18d8b859d42b94edecb0dbf3f764b1367b951d432de251f0252a30450dc87252",
"line": 397,
"raw_line": " 76.47",
"row_label_line": 393,
"row_label": "Average",
"column_header": "phi-4-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-FP8-dynamic/README.md",
"sha256": "18d8b859d42b94edecb0dbf3f764b1367b951d432de251f0252a30450dc87252",
"line": 395,
"raw_line": " | 76.52",
"row_label_line": 393,
"row_label": "Average",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/phi-4-FP8-dynamic@8f62f4b:openllm-v1",
"artifact": "RedHatAI/phi-4-FP8-dynamic",
"artifact_revision": "8f62f4b85ccc1c7fda9f178976c1162f37e43651",
"published_recovery_pct": 99.9
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "MMLU 5-shot (lm-evaluation-harness `openllm`, vLLM)",
"full": 0.803,
"quant_score": 0.803,
"retention": 1,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-FP8-dynamic/blob/8f62f4b85ccc1c7fda9f178976c1162f37e43651/README.md#L337",
"source_id": "quant-redhatai-phi-4-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-FP8-dynamic/README.md",
"sha256": "18d8b859d42b94edecb0dbf3f764b1367b951d432de251f0252a30450dc87252",
"line": 337,
"raw_line": " | 80.30",
"row_label_line": 333,
"row_label": "MMLU (5-shot)",
"column_header": "phi-4-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-FP8-dynamic/README.md",
"sha256": "18d8b859d42b94edecb0dbf3f764b1367b951d432de251f0252a30450dc87252",
"line": 335,
"raw_line": " | 80.30",
"row_label_line": 333,
"row_label": "MMLU (5-shot)",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/phi-4-FP8-dynamic@8f62f4b:openllm-v1",
"artifact": "RedHatAI/phi-4-FP8-dynamic",
"artifact_revision": "8f62f4b85ccc1c7fda9f178976c1162f37e43651",
"published_recovery_pct": 100
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness `openllm`, vLLM)",
"full": 0.9007,
"quant_score": 0.9067,
"retention": 1.0066614855112692,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-FP8-dynamic/blob/8f62f4b85ccc1c7fda9f178976c1162f37e43651/README.md#L357",
"source_id": "quant-redhatai-phi-4-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-FP8-dynamic/README.md",
"sha256": "18d8b859d42b94edecb0dbf3f764b1367b951d432de251f0252a30450dc87252",
"line": 357,
"raw_line": " | 90.67",
"row_label_line": 353,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "phi-4-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-FP8-dynamic/README.md",
"sha256": "18d8b859d42b94edecb0dbf3f764b1367b951d432de251f0252a30450dc87252",
"line": 355,
"raw_line": " | 90.07",
"row_label_line": 353,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/phi-4-FP8-dynamic@8f62f4b:openllm-v1",
"artifact": "RedHatAI/phi-4-FP8-dynamic",
"artifact_revision": "8f62f4b85ccc1c7fda9f178976c1162f37e43651",
"published_recovery_pct": 100.7
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; GPTQ)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7652,
"quant_score": 0.7596,
"retention": 0.9926816518557239,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-quantized.w4a16/blob/e05cd2e17a849b824c85d6acdec3f62be9cd24c1/README.md#L412",
"source_id": "quant-redhatai-phi-4-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w4a16/README.md",
"sha256": "4f92861661bfd43394ec79ff89da6e75fddd72ee24c6ee4bd99016ae3449317b",
"line": 412,
"raw_line": " | 75.96",
"row_label_line": 408,
"row_label": "Average",
"column_header": "phi-4-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w4a16/README.md",
"sha256": "4f92861661bfd43394ec79ff89da6e75fddd72ee24c6ee4bd99016ae3449317b",
"line": 410,
"raw_line": " | 76.52",
"row_label_line": 408,
"row_label": "Average",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/phi-4-quantized.w4a16@e05cd2e:openllm-v1",
"artifact": "RedHatAI/phi-4-quantized.w4a16",
"artifact_revision": "e05cd2e17a849b824c85d6acdec3f62be9cd24c1",
"published_recovery_pct": 99.3
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; GPTQ)",
"metric": "MMLU 5-shot (lm-evaluation-harness `openllm`, vLLM)",
"full": 0.803,
"quant_score": 0.7987,
"retention": 0.994645080946451,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-quantized.w4a16/blob/e05cd2e17a849b824c85d6acdec3f62be9cd24c1/README.md#L352",
"source_id": "quant-redhatai-phi-4-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w4a16/README.md",
"sha256": "4f92861661bfd43394ec79ff89da6e75fddd72ee24c6ee4bd99016ae3449317b",
"line": 352,
"raw_line": " | 79.87",
"row_label_line": 348,
"row_label": "MMLU (5-shot)",
"column_header": "phi-4-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w4a16/README.md",
"sha256": "4f92861661bfd43394ec79ff89da6e75fddd72ee24c6ee4bd99016ae3449317b",
"line": 350,
"raw_line": " | 80.30",
"row_label_line": 348,
"row_label": "MMLU (5-shot)",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/phi-4-quantized.w4a16@e05cd2e:openllm-v1",
"artifact": "RedHatAI/phi-4-quantized.w4a16",
"artifact_revision": "e05cd2e17a849b824c85d6acdec3f62be9cd24c1",
"published_recovery_pct": 99.5
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "INT4 W4A16 (weights only, symmetric per-group, group size 128; GPTQ)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness `openllm`, vLLM)",
"full": 0.9007,
"quant_score": 0.8969,
"retention": 0.9957810591761964,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-quantized.w4a16/blob/e05cd2e17a849b824c85d6acdec3f62be9cd24c1/README.md#L372",
"source_id": "quant-redhatai-phi-4-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w4a16/README.md",
"sha256": "4f92861661bfd43394ec79ff89da6e75fddd72ee24c6ee4bd99016ae3449317b",
"line": 372,
"raw_line": " | 89.69",
"row_label_line": 368,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "phi-4-quantized.w4a16 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w4a16/README.md",
"sha256": "4f92861661bfd43394ec79ff89da6e75fddd72ee24c6ee4bd99016ae3449317b",
"line": 370,
"raw_line": " | 90.07",
"row_label_line": 368,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/phi-4-quantized.w4a16@e05cd2e:openllm-v1",
"artifact": "RedHatAI/phi-4-quantized.w4a16",
"artifact_revision": "e05cd2e17a849b824c85d6acdec3f62be9cd24c1",
"published_recovery_pct": 99.6
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7652,
"quant_score": 0.7635,
"retention": 0.9977783585990591,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-quantized.w8a8/blob/47ae81515aec82176f334e02506eea10dacfddca/README.md#L424",
"source_id": "quant-redhatai-phi-4-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w8a8/README.md",
"sha256": "9b3a49b1f6cc54aa72d578b51c5ee795e1f8566b9de14eb299521cdb00a1b0a0",
"line": 424,
"raw_line": " | 76.35",
"row_label_line": 420,
"row_label": "Average",
"column_header": "phi-4-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w8a8/README.md",
"sha256": "9b3a49b1f6cc54aa72d578b51c5ee795e1f8566b9de14eb299521cdb00a1b0a0",
"line": 422,
"raw_line": " | 76.52",
"row_label_line": 420,
"row_label": "Average",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/phi-4-quantized.w8a8@47ae815:openllm-v1",
"artifact": "RedHatAI/phi-4-quantized.w8a8",
"artifact_revision": "47ae81515aec82176f334e02506eea10dacfddca",
"published_recovery_pct": 99.8
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ)",
"metric": "MMLU 5-shot (lm-evaluation-harness `openllm`, vLLM)",
"full": 0.803,
"quant_score": 0.8039,
"retention": 1.0011207970112082,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-quantized.w8a8/blob/47ae81515aec82176f334e02506eea10dacfddca/README.md#L364",
"source_id": "quant-redhatai-phi-4-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w8a8/README.md",
"sha256": "9b3a49b1f6cc54aa72d578b51c5ee795e1f8566b9de14eb299521cdb00a1b0a0",
"line": 364,
"raw_line": " | 80.39",
"row_label_line": 360,
"row_label": "MMLU (5-shot)",
"column_header": "phi-4-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w8a8/README.md",
"sha256": "9b3a49b1f6cc54aa72d578b51c5ee795e1f8566b9de14eb299521cdb00a1b0a0",
"line": 362,
"raw_line": " | 80.30",
"row_label_line": 360,
"row_label": "MMLU (5-shot)",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/phi-4-quantized.w8a8@47ae815:openllm-v1",
"artifact": "RedHatAI/phi-4-quantized.w8a8",
"artifact_revision": "47ae81515aec82176f334e02506eea10dacfddca",
"published_recovery_pct": 100.1
},
{
"model": "phi-4",
"model_id": "phi-4",
"params_b": 14.7,
"quant": "INT8 W8A8 (weights static per-channel, activations dynamic per-token; SmoothQuant + GPTQ)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness `openllm`, vLLM)",
"full": 0.9007,
"quant_score": 0.903,
"retention": 1.0025535694459866,
"baseline_quant": "BF16 (unquantized microsoft/phi-4, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/phi-4-quantized.w8a8/blob/47ae81515aec82176f334e02506eea10dacfddca/README.md#L384",
"source_id": "quant-redhatai-phi-4-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w8a8/README.md",
"sha256": "9b3a49b1f6cc54aa72d578b51c5ee795e1f8566b9de14eb299521cdb00a1b0a0",
"line": 384,
"raw_line": " | 90.30",
"row_label_line": 380,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "phi-4-quantized.w8a8 (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__phi-4-quantized.w8a8/README.md",
"sha256": "9b3a49b1f6cc54aa72d578b51c5ee795e1f8566b9de14eb299521cdb00a1b0a0",
"line": 382,
"raw_line": " | 90.07",
"row_label_line": 380,
"row_label": "GSM-8K (5-shot, strict-match)",
"column_header": "phi-4"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/phi-4-quantized.w8a8@47ae815:openllm-v1",
"artifact": "RedHatAI/phi-4-quantized.w8a8",
"artifact_revision": "47ae81515aec82176f334e02506eea10dacfddca",
"published_recovery_pct": 100.3
},
{
"model": "Phi-4-mini-instruct",
"model_id": "phi-4-mini",
"params_b": 3.84,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "MMLU with chain-of-thought prompting (lm-evaluation-harness, vLLM)",
"full": 0.6701,
"quant_score": 0.6697,
"retention": 0.9994030741680345,
"baseline_quant": "BF16 (unquantized microsoft/Phi-4-mini-instruct, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Phi-4-mini-instruct-FP8-dynamic/blob/5450d52149bf44e81cbbd30b2f02762947845bb3/README.md#L279",
"source_id": "quant-redhatai-phi-4-mini-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-mini-instruct-FP8-dynamic/README.md",
"sha256": "c2d433fb37e393fd5b2ac3448c9c942720062e189fbe6d4f77a7a2ba9e6d8684",
"line": 279,
"raw_line": " | 66.97",
"row_label_line": 275,
"row_label": "MMLU CoT",
"column_header": "Phi-4-mini-instruct-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-mini-instruct-FP8-dynamic/README.md",
"sha256": "c2d433fb37e393fd5b2ac3448c9c942720062e189fbe6d4f77a7a2ba9e6d8684",
"line": 277,
"raw_line": " | 67.01",
"row_label_line": 275,
"row_label": "MMLU CoT",
"column_header": "Phi-4-mini-instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Phi-4-mini-instruct-FP8-dynamic@5450d52:mini",
"artifact": "RedHatAI/Phi-4-mini-instruct-FP8-dynamic",
"artifact_revision": "5450d52149bf44e81cbbd30b2f02762947845bb3",
"published_recovery_pct": 99.9
},
{
"model": "Phi-4-mini-instruct",
"model_id": "phi-4-mini",
"params_b": 3.84,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "MMLU-Pro (lm-evaluation-harness, vLLM)",
"full": 0.4675,
"quant_score": 0.456,
"retention": 0.9754010695187165,
"baseline_quant": "BF16 (unquantized microsoft/Phi-4-mini-instruct, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Phi-4-mini-instruct-FP8-dynamic/blob/5450d52149bf44e81cbbd30b2f02762947845bb3/README.md#L289",
"source_id": "quant-redhatai-phi-4-mini-instruct-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-mini-instruct-FP8-dynamic/README.md",
"sha256": "c2d433fb37e393fd5b2ac3448c9c942720062e189fbe6d4f77a7a2ba9e6d8684",
"line": 289,
"raw_line": " | 45.60",
"row_label_line": 285,
"row_label": "MMLU-Pro",
"column_header": "Phi-4-mini-instruct-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-mini-instruct-FP8-dynamic/README.md",
"sha256": "c2d433fb37e393fd5b2ac3448c9c942720062e189fbe6d4f77a7a2ba9e6d8684",
"line": 287,
"raw_line": " | 46.75",
"row_label_line": 285,
"row_label": "MMLU-Pro",
"column_header": "Phi-4-mini-instruct"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Phi-4-mini-instruct-FP8-dynamic@5450d52:mini",
"artifact": "RedHatAI/Phi-4-mini-instruct-FP8-dynamic",
"artifact_revision": "5450d52149bf44e81cbbd30b2f02762947845bb3",
"published_recovery_pct": 97.5
},
{
"model": "Phi-4-reasoning",
"model_id": "phi-4-reasoning",
"params_b": 14.7,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "AIME 2025 (lighteval, vLLM, reasoning parser)",
"full": 0.6125,
"quant_score": 0.6458,
"retention": 1.0543673469387753,
"baseline_quant": "BF16 (unquantized microsoft/Phi-4-reasoning, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Phi-4-reasoning-FP8-dynamic/blob/10256f614f546d20fddd470592e55e59e6a6e2e4/README.md#L212",
"source_id": "quant-redhatai-phi-4-reasoning-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 212,
"raw_line": " | 64.58",
"row_label_line": 208,
"row_label": "AIME25",
"column_header": "Phi-4-reasoning-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 210,
"raw_line": " | 61.25",
"row_label_line": 208,
"row_label": "AIME25",
"column_header": "Phi-4-reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline differs from Microsoft-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Phi-4-reasoning-FP8-dynamic@10256f6:r",
"artifact": "RedHatAI/Phi-4-reasoning-FP8-dynamic",
"artifact_revision": "10256f614f546d20fddd470592e55e59e6a6e2e4",
"published_recovery_pct": 105.4
},
{
"model": "Phi-4-reasoning",
"model_id": "phi-4-reasoning",
"params_b": 14.7,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "GPQA Diamond (lighteval, vLLM)",
"full": 0.6465,
"quant_score": 0.665,
"retention": 1.02861562258314,
"baseline_quant": "BF16 (unquantized microsoft/Phi-4-reasoning, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Phi-4-reasoning-FP8-dynamic/blob/10256f614f546d20fddd470592e55e59e6a6e2e4/README.md#L222",
"source_id": "quant-redhatai-phi-4-reasoning-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 222,
"raw_line": " | 66.50",
"row_label_line": 218,
"row_label": "GPQA Diamond",
"column_header": "Phi-4-reasoning-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 220,
"raw_line": " | 64.65",
"row_label_line": 218,
"row_label": "GPQA Diamond",
"column_header": "Phi-4-reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline differs from Microsoft-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Phi-4-reasoning-FP8-dynamic@10256f6:r",
"artifact": "RedHatAI/Phi-4-reasoning-FP8-dynamic",
"artifact_revision": "10256f614f546d20fddd470592e55e59e6a6e2e4",
"published_recovery_pct": 102.9
},
{
"model": "Phi-4-reasoning",
"model_id": "phi-4-reasoning",
"params_b": 14.7,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "MATH-500 (lighteval, vLLM)",
"full": 0.9001,
"quant_score": 0.886,
"retention": 0.9843350738806798,
"baseline_quant": "BF16 (unquantized microsoft/Phi-4-reasoning, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Phi-4-reasoning-FP8-dynamic/blob/10256f614f546d20fddd470592e55e59e6a6e2e4/README.md#L232",
"source_id": "quant-redhatai-phi-4-reasoning-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 232,
"raw_line": " | 88.60",
"row_label_line": 228,
"row_label": "Math 500",
"column_header": "Phi-4-reasoning-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 230,
"raw_line": " | 90.01",
"row_label_line": 228,
"row_label": "Math 500",
"column_header": "Phi-4-reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline differs from Microsoft-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Phi-4-reasoning-FP8-dynamic@10256f6:r",
"artifact": "RedHatAI/Phi-4-reasoning-FP8-dynamic",
"artifact_revision": "10256f614f546d20fddd470592e55e59e6a6e2e4",
"published_recovery_pct": 98.4
},
{
"model": "Phi-4-reasoning",
"model_id": "phi-4-reasoning",
"params_b": 14.7,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "MMLU-Pro 0-shot chat (lm-evaluation-harness `mmlu_pro_chat`, vLLM)",
"full": 0.7649,
"quant_score": 0.7685,
"retention": 1.0047064975813833,
"baseline_quant": "BF16 (unquantized microsoft/Phi-4-reasoning, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Phi-4-reasoning-FP8-dynamic/blob/10256f614f546d20fddd470592e55e59e6a6e2e4/README.md#L242",
"source_id": "quant-redhatai-phi-4-reasoning-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 242,
"raw_line": " | 76.85",
"row_label_line": 238,
"row_label": "MMLU-Pro",
"column_header": "Phi-4-reasoning-FP8-dynamic (this model)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Phi-4-reasoning-FP8-dynamic/README.md",
"sha256": "ca0169e272dcc6780a69e1e344b97f400829ecf8c6a7af0edc3ed91fc5cd668d",
"line": 240,
"raw_line": " | 76.49",
"row_label_line": 238,
"row_label": "MMLU-Pro",
"column_header": "Phi-4-reasoning"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Baseline differs from Microsoft-reported values, i.e. re-measured. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Phi-4-reasoning-FP8-dynamic@10256f6:r",
"artifact": "RedHatAI/Phi-4-reasoning-FP8-dynamic",
"artifact_revision": "10256f614f546d20fddd470592e55e59e6a6e2e4",
"published_recovery_pct": 100.5
},
{
"model": "QwQ-32B",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (FP8 weights per-channel, activations per-token; llm-compressor FP8_DYNAMIC)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1, MATH-500 pass@1, GPQA Diamond pass@1)",
"full": 0.8025,
"quant_score": 0.8005,
"retention": 0.9975077881619938,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-FP8-dynamic/blob/c4f1385172c9b31fd151820312aedd6a20fe6830/README.md#L145",
"source_id": "quant-redhatai-qwq-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 145,
"raw_line": " | 80.05 | ",
"row_label_line": 143,
"row_label": "Average Score",
"column_header": "neuralmagic/QwQ-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 144,
"raw_line": "80.25 | ",
"row_label_line": 143,
"row_label": "Average Score",
"column_header": "Qwen/QwQ-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B is not in data/models.json (model_id null); params_b from the model name. The card does not describe the evaluation harness. Repository declares license: mit (the upstream Qwen repo is Apache-2.0); the MIT declaration is relied on for this card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks: AIME 2024, MATH-500, GPQA Diamond",
"study": "RedHatAI/QwQ-32B-FP8-dynamic@c4f1385:reasoning",
"artifact": "RedHatAI/QwQ-32B-FP8-dynamic",
"artifact_revision": "c4f1385172c9b31fd151820312aedd6a20fe6830",
"published_recovery_pct": 99.75
},
{
"model": "QwQ-32B",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (FP8 weights per-channel, activations per-token; llm-compressor FP8_DYNAMIC)",
"metric": "AIME 2024 pass@1",
"full": 0.7866,
"quant_score": 0.794,
"retention": 1.0094075769132977,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-FP8-dynamic/blob/c4f1385172c9b31fd151820312aedd6a20fe6830/README.md#L127",
"source_id": "quant-redhatai-qwq-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 127,
"raw_line": "79.40 | ",
"row_label_line": 125,
"row_label": "AIME 2024 (pass@1)",
"column_header": "neuralmagic/QwQ-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 126,
"raw_line": "78.66 | ",
"row_label_line": 125,
"row_label": "AIME 2024 (pass@1)",
"column_header": "Qwen/QwQ-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B is not in data/models.json (model_id null); params_b from the model name. The card does not describe the evaluation harness. Repository declares license: mit (the upstream Qwen repo is Apache-2.0); the MIT declaration is relied on for this card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-FP8-dynamic@c4f1385:reasoning",
"artifact": "RedHatAI/QwQ-32B-FP8-dynamic",
"artifact_revision": "c4f1385172c9b31fd151820312aedd6a20fe6830",
"published_recovery_pct": 100.94
},
{
"model": "QwQ-32B",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (FP8 weights per-channel, activations per-token; llm-compressor FP8_DYNAMIC)",
"metric": "MATH-500 pass@1",
"full": 0.9739,
"quant_score": 0.9744,
"retention": 1.000513399733032,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-FP8-dynamic/blob/c4f1385172c9b31fd151820312aedd6a20fe6830/README.md#L133",
"source_id": "quant-redhatai-qwq-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 133,
"raw_line": "97.44 | ",
"row_label_line": 131,
"row_label": "MATH-500 (pass@1)",
"column_header": "neuralmagic/QwQ-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 132,
"raw_line": "97.39 | ",
"row_label_line": 131,
"row_label": "MATH-500 (pass@1)",
"column_header": "Qwen/QwQ-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B is not in data/models.json (model_id null); params_b from the model name. The card does not describe the evaluation harness. Repository declares license: mit (the upstream Qwen repo is Apache-2.0); the MIT declaration is relied on for this card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-FP8-dynamic@c4f1385:reasoning",
"artifact": "RedHatAI/QwQ-32B-FP8-dynamic",
"artifact_revision": "c4f1385172c9b31fd151820312aedd6a20fe6830",
"published_recovery_pct": 100.05
},
{
"model": "QwQ-32B",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (FP8 weights per-channel, activations per-token; llm-compressor FP8_DYNAMIC)",
"metric": "GPQA Diamond pass@1",
"full": 0.6472,
"quant_score": 0.6321,
"retention": 0.9766687268232386,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-FP8-dynamic/blob/c4f1385172c9b31fd151820312aedd6a20fe6830/README.md#L139",
"source_id": "quant-redhatai-qwq-32b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 139,
"raw_line": "63.21 | ",
"row_label_line": 137,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "neuralmagic/QwQ-32B-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-FP8-dynamic/README.md",
"sha256": "bbe3ae92ef695a6669b68733a109d310b067742aa67f56bf690ab86f8a5771ea",
"line": 138,
"raw_line": "64.72 | ",
"row_label_line": 137,
"row_label": "GPQA Diamond (pass@1)",
"column_header": "Qwen/QwQ-32B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B is not in data/models.json (model_id null); params_b from the model name. The card does not describe the evaluation harness. Repository declares license: mit (the upstream Qwen repo is Apache-2.0); the MIT declaration is relied on for this card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-FP8-dynamic@c4f1385:reasoning",
"artifact": "RedHatAI/QwQ-32B-FP8-dynamic",
"artifact_revision": "c4f1385172c9b31fd151820312aedd6a20fe6830",
"published_recovery_pct": 97.66
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.772,
"quant_score": 0.7708,
"retention": 0.9984455958549222,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-FP8-dynamic/blob/de33103179314a2c626b998043238ce21700e64d/README.md#L153",
"source_id": "quant-redhatai-qwq-32b-preview-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 153,
"raw_line": "| **Average Score** | **77.20** | **77.08** |",
"row_label_line": 153,
"row_label": "Average Score",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 153,
"raw_line": "| **Average Score** | **77.20** | **77.08** |",
"row_label_line": 153,
"row_label": "Average Score",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/QwQ-32B-Preview-FP8-dynamic@de33103:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-FP8-dynamic",
"artifact_revision": "de33103179314a2c626b998043238ce21700e64d",
"published_recovery_pct": 99.84
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8309,
"quant_score": 0.8218,
"retention": 0.9890480202190396,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-FP8-dynamic/blob/de33103179314a2c626b998043238ce21700e64d/README.md#L148",
"source_id": "quant-redhatai-qwq-32b-preview-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 148,
"raw_line": "| GSM8K (Strict-Match, 5-shot) | 83.09 | 82.18 |",
"row_label_line": 148,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 148,
"raw_line": "| GSM8K (Strict-Match, 5-shot) | 83.09 | 82.18 |",
"row_label_line": 148,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-FP8-dynamic@de33103:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-FP8-dynamic",
"artifact_revision": "de33103179314a2c626b998043238ce21700e64d"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.8267,
"quant_score": 0.8267,
"retention": 1,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-FP8-dynamic/blob/de33103179314a2c626b998043238ce21700e64d/README.md#L150",
"source_id": "quant-redhatai-qwq-32b-preview-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 150,
"raw_line": "| MMLU (Acc, 5-shot) | 82.67 | 82.67 |",
"row_label_line": 150,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 150,
"raw_line": "| MMLU (Acc, 5-shot) | 82.67 | 82.67 |",
"row_label_line": 150,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-FP8-dynamic@de33103:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-FP8-dynamic",
"artifact_revision": "de33103179314a2c626b998043238ce21700e64d"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.3018,
"quant_score": 0.298,
"retention": 0.9874088800530152,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-FP8-dynamic/blob/de33103179314a2c626b998043238ce21700e64d/README.md#L166",
"source_id": "quant-redhatai-qwq-32b-preview-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 166,
"raw_line": "| **Average Score** | **30.18** | **29.80** |",
"row_label_line": 166,
"row_label": "Average Score",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 166,
"raw_line": "| **Average Score** | **30.18** | **29.80** |",
"row_label_line": 166,
"row_label": "Average Score",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/QwQ-32B-Preview-FP8-dynamic@de33103:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-FP8-dynamic",
"artifact_revision": "de33103179314a2c626b998043238ce21700e64d",
"published_recovery_pct": 98.74
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot inst-and-prompt-level strict acc (OpenLLM v2 config)",
"full": 0.4234,
"quant_score": 0.4048,
"retention": 0.9560699102503541,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-FP8-dynamic/blob/de33103179314a2c626b998043238ce21700e64d/README.md#L160",
"source_id": "quant-redhatai-qwq-32b-preview-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 160,
"raw_line": "| IFEval (Inst-and-Prompt Level Strict Acc, 0-shot) | 42.34 | 40.48 |",
"row_label_line": 160,
"row_label": "IFEval (Inst-and-Prompt Level Strict Acc, 0-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 160,
"raw_line": "| IFEval (Inst-and-Prompt Level Strict Acc, 0-shot) | 42.34 | 40.48 |",
"row_label_line": 160,
"row_label": "IFEval (Inst-and-Prompt Level Strict Acc, 0-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-FP8-dynamic@de33103:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-FP8-dynamic",
"artifact_revision": "de33103179314a2c626b998043238ce21700e64d"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "FP8 W8A8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published)",
"full": 0.52,
"quant_score": 0.5162,
"retention": 0.9926923076923077,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-FP8-dynamic/blob/de33103179314a2c626b998043238ce21700e64d/README.md#L165",
"source_id": "quant-redhatai-qwq-32b-preview-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 165,
"raw_line": "| MMLU-Pro (Acc, 5-shot) | 52.00 | 51.62 |",
"row_label_line": 165,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-FP8-dynamic"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-FP8-dynamic/README.md",
"sha256": "7171bc1fd1b6c4838b22497db89003db71de3eedfa84882b7f374c0fb455d491",
"line": 165,
"raw_line": "| MMLU-Pro (Acc, 5-shot) | 52.00 | 51.62 |",
"row_label_line": 165,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-FP8-dynamic@de33103:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-FP8-dynamic",
"artifact_revision": "de33103179314a2c626b998043238ce21700e64d"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.772,
"quant_score": 0.7587,
"retention": 0.9827720207253886,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w4a16/blob/be9a600807f053d20438b4511466d8eb0fef909c/README.md#L218",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 218,
"raw_line": "| **Average Score** | **77.20** | **75.87** |",
"row_label_line": 218,
"row_label": "Average Score",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w4a16"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 218,
"raw_line": "| **Average Score** | **77.20** | **75.87** |",
"row_label_line": 218,
"row_label": "Average Score",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w4a16@be9a600:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w4a16",
"artifact_revision": "be9a600807f053d20438b4511466d8eb0fef909c",
"published_recovery_pct": 98.28
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8309,
"quant_score": 0.7983,
"retention": 0.9607654350704055,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w4a16/blob/be9a600807f053d20438b4511466d8eb0fef909c/README.md#L213",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 213,
"raw_line": "| GSM8K (Strict-Match, 5-shot) | 83.09 | 79.83 |",
"row_label_line": 213,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w4a16"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 213,
"raw_line": "| GSM8K (Strict-Match, 5-shot) | 83.09 | 79.83 |",
"row_label_line": 213,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w4a16@be9a600:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w4a16",
"artifact_revision": "be9a600807f053d20438b4511466d8eb0fef909c"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.8267,
"quant_score": 0.8158,
"retention": 0.9868150477803315,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w4a16/blob/be9a600807f053d20438b4511466d8eb0fef909c/README.md#L215",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 215,
"raw_line": "| MMLU (Acc, 5-shot) | 82.67 | 81.58 |",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w4a16"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 215,
"raw_line": "| MMLU (Acc, 5-shot) | 82.67 | 81.58 |",
"row_label_line": 215,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w4a16@be9a600:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w4a16",
"artifact_revision": "be9a600807f053d20438b4511466d8eb0fef909c"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.3018,
"quant_score": 0.3025,
"retention": 1.0023194168323393,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w4a16/blob/be9a600807f053d20438b4511466d8eb0fef909c/README.md#L231",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 231,
"raw_line": "| **Average Score** | **30.18** | **30.25** |",
"row_label_line": 231,
"row_label": "Average Score",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w4a16"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 231,
"raw_line": "| **Average Score** | **30.18** | **30.25** |",
"row_label_line": 231,
"row_label": "Average Score",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w4a16@be9a600:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w4a16",
"artifact_revision": "be9a600807f053d20438b4511466d8eb0fef909c",
"published_recovery_pct": 100.23
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "IFEval 0-shot inst-and-prompt-level strict acc (OpenLLM v2 config)",
"full": 0.4234,
"quant_score": 0.4109,
"retention": 0.9704770902220122,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w4a16/blob/be9a600807f053d20438b4511466d8eb0fef909c/README.md#L225",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 225,
"raw_line": "| IFEval (Inst-and-Prompt Level Strict Acc, 0-shot) | 42.34 | 41.09 |",
"row_label_line": 225,
"row_label": "IFEval (Inst-and-Prompt Level Strict Acc, 0-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w4a16"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 225,
"raw_line": "| IFEval (Inst-and-Prompt Level Strict Acc, 0-shot) | 42.34 | 41.09 |",
"row_label_line": 225,
"row_label": "IFEval (Inst-and-Prompt Level Strict Acc, 0-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w4a16@be9a600:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w4a16",
"artifact_revision": "be9a600807f053d20438b4511466d8eb0fef909c"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published)",
"full": 0.52,
"quant_score": 0.4987,
"retention": 0.9590384615384615,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w4a16/blob/be9a600807f053d20438b4511466d8eb0fef909c/README.md#L230",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 230,
"raw_line": "| MMLU-Pro (Acc, 5-shot) | 52.00 | 49.87 |",
"row_label_line": 230,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w4a16"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w4a16/README.md",
"sha256": "444c28b820bd08f67179d830e496a73d737291480707645314e796ebafe7d0c5",
"line": 230,
"raw_line": "| MMLU-Pro (Acc, 5-shot) | 52.00 | 49.87 |",
"row_label_line": 230,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w4a16@be9a600:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w4a16",
"artifact_revision": "be9a600807f053d20438b4511466d8eb0fef909c"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT8 W8A8 (GPTQ via llm-compressor)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.772,
"quant_score": 0.7649,
"retention": 0.99080310880829,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w8a8/blob/bf17e01cb466e31ac49d5ae188e0b77121945cf9/README.md#L190",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 190,
"raw_line": "| **Average Score** | **77.20** | **76.49** |",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 190,
"raw_line": "| **Average Score** | **77.20** | **76.49** |",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w8a8@bf17e01:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w8a8",
"artifact_revision": "bf17e01cb466e31ac49d5ae188e0b77121945cf9",
"published_recovery_pct": 99.08
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT8 W8A8 (GPTQ via llm-compressor)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8309,
"quant_score": 0.7991,
"retention": 0.9617282464797205,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w8a8/blob/bf17e01cb466e31ac49d5ae188e0b77121945cf9/README.md#L185",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 185,
"raw_line": "| GSM8K (Strict-Match, 5-shot) | 83.09 | 79.91 |",
"row_label_line": 185,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 185,
"raw_line": "| GSM8K (Strict-Match, 5-shot) | 83.09 | 79.91 |",
"row_label_line": 185,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w8a8@bf17e01:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w8a8",
"artifact_revision": "bf17e01cb466e31ac49d5ae188e0b77121945cf9"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT8 W8A8 (GPTQ via llm-compressor)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.8267,
"quant_score": 0.8224,
"retention": 0.9947985968307729,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w8a8/blob/bf17e01cb466e31ac49d5ae188e0b77121945cf9/README.md#L187",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 187,
"raw_line": "| MMLU (Acc, 5-shot) | 82.67 | 82.24 |",
"row_label_line": 187,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 187,
"raw_line": "| MMLU (Acc, 5-shot) | 82.67 | 82.24 |",
"row_label_line": 187,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w8a8@bf17e01:openllm-v1",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w8a8",
"artifact_revision": "bf17e01cb466e31ac49d5ae188e0b77121945cf9"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT8 W8A8 (GPTQ via llm-compressor)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.3018,
"quant_score": 0.3076,
"retention": 1.01921802518224,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w8a8/blob/bf17e01cb466e31ac49d5ae188e0b77121945cf9/README.md#L203",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 203,
"raw_line": "| **Average Score** | **30.18** | **30.76** |",
"row_label_line": 203,
"row_label": "Average Score",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 203,
"raw_line": "| **Average Score** | **30.18** | **30.76** |",
"row_label_line": 203,
"row_label": "Average Score",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w8a8@bf17e01:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w8a8",
"artifact_revision": "bf17e01cb466e31ac49d5ae188e0b77121945cf9",
"published_recovery_pct": 101.92
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT8 W8A8 (GPTQ via llm-compressor)",
"metric": "IFEval 0-shot inst-and-prompt-level strict acc (OpenLLM v2 config)",
"full": 0.4234,
"quant_score": 0.4349,
"retention": 1.0271610769957487,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w8a8/blob/bf17e01cb466e31ac49d5ae188e0b77121945cf9/README.md#L197",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 197,
"raw_line": "| IFEval (Inst-and-Prompt Level Strict Acc, 0-shot) | 42.34 | 43.49 |",
"row_label_line": 197,
"row_label": "IFEval (Inst-and-Prompt Level Strict Acc, 0-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 197,
"raw_line": "| IFEval (Inst-and-Prompt Level Strict Acc, 0-shot) | 42.34 | 43.49 |",
"row_label_line": 197,
"row_label": "IFEval (Inst-and-Prompt Level Strict Acc, 0-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w8a8@bf17e01:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w8a8",
"artifact_revision": "bf17e01cb466e31ac49d5ae188e0b77121945cf9"
},
{
"model": "QwQ-32B-Preview",
"model_id": null,
"params_b": 32,
"quant": "INT8 W8A8 (GPTQ via llm-compressor)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published)",
"full": 0.52,
"quant_score": 0.514,
"retention": 0.9884615384615385,
"baseline_quant": "BF16 (unquantized Qwen/QwQ-32B-Preview, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/QwQ-32B-Preview-quantized.w8a8/blob/bf17e01cb466e31ac49d5ae188e0b77121945cf9/README.md#L202",
"source_id": "quant-redhatai-qwq-32b-preview-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 202,
"raw_line": "| MMLU-Pro (Acc, 5-shot) | 52.00 | 51.4 |",
"row_label_line": 202,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "neuralmagic-ent/QwQ-32B-Preview-quantized.w8a8"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__QwQ-32B-Preview-quantized.w8a8/README.md",
"sha256": "3538fcc406a2803746b5bc0f35b9a2b2dc90e12de1a7f5c6da1f899309777d5f",
"line": 202,
"raw_line": "| MMLU-Pro (Acc, 5-shot) | 52.00 | 51.4 |",
"row_label_line": 202,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "Qwen/QwQ-32B-Preview"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. QwQ-32B-Preview is a different checkpoint from QwQ-32B and is not in data/models.json. Per-row recovery not published; only the suite-average recovery row is published and was checked. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/QwQ-32B-Preview-quantized.w8a8@bf17e01:openllm-v2",
"artifact": "RedHatAI/QwQ-32B-Preview-quantized.w8a8",
"artifact_revision": "bf17e01cb466e31ac49d5ae188e0b77121945cf9"
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1 over 64 samples, MATH-500 pass@1 over 4, GPQA Diamond pass@1 over 8; LightEval, vLLM)",
"full": 0.5861,
"quant_score": 0.589,
"retention": 1.0049479610987886,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-FP8-dynamic/blob/01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084/README.md#L206",
"source_id": "quant-redhatai-smollm3-3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 206,
"raw_line": " 58.90",
"row_label_line": 202,
"row_label": "Average",
"column_header": "RedHatAI/SmolLM3-3B-FP8-dynamic (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 204,
"raw_line": " | 58.61",
"row_label_line": 202,
"row_label": "Average",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks",
"study": "RedHatAI/SmolLM3-3B-FP8-dynamic@01a88bc:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-FP8-dynamic",
"artifact_revision": "01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084",
"published_recovery_pct": 100.5
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "AIME 2024 pass@1 (64 samples; LightEval, vLLM)",
"full": 0.4531,
"quant_score": 0.475,
"retention": 1.0483337011697196,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-FP8-dynamic/blob/01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084/README.md#L176",
"source_id": "quant-redhatai-smollm3-3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 176,
"raw_line": " | 47.50",
"row_label_line": 172,
"row_label": "AIME24 (pass@1:64)",
"column_header": "RedHatAI/SmolLM3-3B-FP8-dynamic (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 174,
"raw_line": " | 45.31",
"row_label_line": 172,
"row_label": "AIME24 (pass@1:64)",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/SmolLM3-3B-FP8-dynamic@01a88bc:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-FP8-dynamic",
"artifact_revision": "01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084",
"published_recovery_pct": 104.83
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "MATH-500 pass@1 (4 samples; LightEval, vLLM)",
"full": 0.893,
"quant_score": 0.883,
"retention": 0.9888017917133258,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-FP8-dynamic/blob/01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084/README.md#L186",
"source_id": "quant-redhatai-smollm3-3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 186,
"raw_line": " | 88.30",
"row_label_line": 182,
"row_label": "MATH-500 (pass@1:4)",
"column_header": "RedHatAI/SmolLM3-3B-FP8-dynamic (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 184,
"raw_line": " | 89.30",
"row_label_line": 182,
"row_label": "MATH-500 (pass@1:4)",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/SmolLM3-3B-FP8-dynamic@01a88bc:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-FP8-dynamic",
"artifact_revision": "01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084",
"published_recovery_pct": 98.88
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token)",
"metric": "GPQA Diamond pass@1 (8 samples; LightEval, vLLM)",
"full": 0.4122,
"quant_score": 0.4091,
"retention": 0.9924793789422609,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-FP8-dynamic/blob/01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084/README.md#L196",
"source_id": "quant-redhatai-smollm3-3b-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 196,
"raw_line": " | 40.91",
"row_label_line": 192,
"row_label": "GPQA-Diamond (pass@1:8)",
"column_header": "RedHatAI/SmolLM3-3B-FP8-dynamic (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-FP8-dynamic/README.md",
"sha256": "f762e398f14883a7a90f06c83282d8d04056aed3802a3ab641f7fe4ddd3b9387",
"line": 194,
"raw_line": " | 41.22",
"row_label_line": 192,
"row_label": "GPQA-Diamond (pass@1:8)",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/SmolLM3-3B-FP8-dynamic@01a88bc:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-FP8-dynamic",
"artifact_revision": "01a88bc017932dbe4c7f0a4fc1db0bb72a7ab084",
"published_recovery_pct": 99.25
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "Reasoning average of 3 tasks (AIME 2024 pass@1 over 64 samples, MATH-500 pass@1 over 4, GPQA Diamond pass@1 over 8; LightEval, vLLM)",
"full": 0.5861,
"quant_score": 0.5623,
"retention": 0.9593925951202867,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-quantized.w4a16/blob/8d782c9fd7495f5d2add92b395d218b906e67678/README.md#L277",
"source_id": "quant-redhatai-smollm3-3b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 277,
"raw_line": " | 56.23",
"row_label_line": 273,
"row_label": "Average",
"column_header": "RedHatAI/SmolLM3-3B-quantized.w4a16 (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 275,
"raw_line": " | 58.61",
"row_label_line": 273,
"row_label": "Average",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 3 tasks",
"study": "RedHatAI/SmolLM3-3B-quantized.w4a16@8d782c9:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-quantized.w4a16",
"artifact_revision": "8d782c9fd7495f5d2add92b395d218b906e67678",
"published_recovery_pct": 95.94
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "AIME 2024 pass@1 (64 samples; LightEval, vLLM)",
"full": 0.4531,
"quant_score": 0.3927,
"retention": 0.8666960935775768,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-quantized.w4a16/blob/8d782c9fd7495f5d2add92b395d218b906e67678/README.md#L247",
"source_id": "quant-redhatai-smollm3-3b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 247,
"raw_line": " | 39.27",
"row_label_line": 243,
"row_label": "AIME24 (pass@1:64)",
"column_header": "RedHatAI/SmolLM3-3B-quantized.w4a16 (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 245,
"raw_line": " | 45.31",
"row_label_line": 243,
"row_label": "AIME24 (pass@1:64)",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/SmolLM3-3B-quantized.w4a16@8d782c9:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-quantized.w4a16",
"artifact_revision": "8d782c9fd7495f5d2add92b395d218b906e67678",
"published_recovery_pct": 86.67
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "MATH-500 pass@1 (4 samples; LightEval, vLLM)",
"full": 0.893,
"quant_score": 0.8755,
"retention": 0.9804031354983201,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-quantized.w4a16/blob/8d782c9fd7495f5d2add92b395d218b906e67678/README.md#L257",
"source_id": "quant-redhatai-smollm3-3b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 257,
"raw_line": " | 87.55",
"row_label_line": 253,
"row_label": "MATH-500 (pass@1:4)",
"column_header": "RedHatAI/SmolLM3-3B-quantized.w4a16 (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 255,
"raw_line": " | 89.30",
"row_label_line": 253,
"row_label": "MATH-500 (pass@1:4)",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/SmolLM3-3B-quantized.w4a16@8d782c9:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-quantized.w4a16",
"artifact_revision": "8d782c9fd7495f5d2add92b395d218b906e67678",
"published_recovery_pct": 98.04
},
{
"model": "SmolLM3-3B",
"model_id": "smollm3-3b",
"params_b": 3.08,
"quant": "INT4 W4A16 (weights only, group size 128; GPTQ)",
"metric": "GPQA Diamond pass@1 (8 samples; LightEval, vLLM)",
"full": 0.4122,
"quant_score": 0.4186,
"retention": 1.0155264434740416,
"baseline_quant": "BF16 (unquantized HuggingFaceTB/SmolLM3-3B, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/SmolLM3-3B-quantized.w4a16/blob/8d782c9fd7495f5d2add92b395d218b906e67678/README.md#L267",
"source_id": "quant-redhatai-smollm3-3b-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 267,
"raw_line": " | 41.86",
"row_label_line": 263,
"row_label": "GPQA-Diamond (pass@1:8)",
"column_header": "RedHatAI/SmolLM3-3B-quantized.w4a16 (this model)",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__SmolLM3-3B-quantized.w4a16/README.md",
"sha256": "b14a3a042bb48c773eb31a6b5c2a5c64b07014341dd6598ca75bd3b261d89f8e",
"line": 265,
"raw_line": " | 41.22",
"row_label_line": 263,
"row_label": "GPQA-Diamond (pass@1:8)",
"column_header": "HuggingFaceTB/SmolLM3-3B"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Generation: temperature 0.6, top_p 0.95, max_new_tokens 32768. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/SmolLM3-3B-quantized.w4a16@8d782c9:reasoning",
"artifact": "RedHatAI/SmolLM3-3B-quantized.w4a16",
"artifact_revision": "8d782c9fd7495f5d2add92b395d218b906e67678",
"published_recovery_pct": 101.55
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "IFEval 0-shot prompt-level strict (without thinking)",
"full": 0.8996,
"quant_score": 0.8934,
"retention": 0.993108048021343,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L155",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 155,
"raw_line": " | 89.34 | ",
"row_label_line": 153,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 154,
"raw_line": " 89.96 | ",
"row_label_line": 153,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 99.3
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MMLU-Pro 0-shot custom-extract (without thinking)",
"full": 0.8347,
"quant_score": 0.8326,
"retention": 0.9974841260333054,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L174",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 174,
"raw_line": " 83.26 | ",
"row_label_line": 172,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 173,
"raw_line": " 83.47 | ",
"row_label_line": 172,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 99.7
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MATH-500 0-shot pass@1 (without thinking)",
"full": 0.848,
"quant_score": 0.8593,
"retention": 1.0133254716981133,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L180",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 180,
"raw_line": " 85.93 | ",
"row_label_line": 178,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 179,
"raw_line": " 84.80 | ",
"row_label_line": 178,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 101.3
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (without thinking)",
"full": 0.8,
"quant_score": 0.8,
"retention": 1,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L186",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 186,
"raw_line": " 80.00 | ",
"row_label_line": 184,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 185,
"raw_line": " 80.00 | ",
"row_label_line": 184,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 100
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "GPQA Diamond 0-shot pass@1 (without thinking)",
"full": 0.732,
"quant_score": 0.7475,
"retention": 1.0211748633879782,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L192",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 192,
"raw_line": " 74.75 | ",
"row_label_line": 190,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 191,
"raw_line": " 73.20 | ",
"row_label_line": 190,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 102.1
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "LiveCodeBench v6 0-shot pass@1 (without thinking)",
"full": 0.7448,
"quant_score": 0.739,
"retention": 0.9922126745435017,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L199",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 199,
"raw_line": " 73.90 | ",
"row_label_line": 197,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 198,
"raw_line": " 74.48 | ",
"row_label_line": 197,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 99.2
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "IFEval 0-shot prompt-level strict (with thinking)",
"full": 0.9439,
"quant_score": 0.9507,
"retention": 1.0072041529823073,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L222",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 222,
"raw_line": " 95.07 | ",
"row_label_line": 220,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 221,
"raw_line": " 94.39 | ",
"row_label_line": 220,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 100.7
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MMLU-Pro 0-shot custom-extract (with thinking)",
"full": 0.8519,
"quant_score": 0.8507,
"retention": 0.998591383965254,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L241",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 241,
"raw_line": " 85.07 | ",
"row_label_line": 239,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 240,
"raw_line": " 85.19 | ",
"row_label_line": 239,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 99.9
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MATH-500 0-shot pass@1 (with thinking)",
"full": 0.8587,
"quant_score": 0.8593,
"retention": 1.0006987306393387,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L247",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 247,
"raw_line": " 85.93 | ",
"row_label_line": 245,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 246,
"raw_line": " 85.87 | ",
"row_label_line": 245,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 100.1
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (with thinking)",
"full": 0.8875,
"quant_score": 0.9167,
"retention": 1.0329014084507044,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L253",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 253,
"raw_line": " 91.67 | ",
"row_label_line": 251,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 252,
"raw_line": " 88.75 | ",
"row_label_line": 251,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 103.3
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "GPQA Diamond 0-shot pass@1 (with thinking)",
"full": 0.8081,
"quant_score": 0.8013,
"retention": 0.9915851998515034,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L259",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 259,
"raw_line": " 80.13 | ",
"row_label_line": 257,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 258,
"raw_line": " 80.81 | ",
"row_label_line": 257,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 99.2
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "LiveCodeBench v6 0-shot pass@1 (with thinking)",
"full": 0.779,
"quant_score": 0.76,
"retention": 0.975609756097561,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic/blob/ed35d7abe5d940da41b4ff06eb482feb0be8cb44/README.md#L266",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 266,
"raw_line": " 76.00 | ",
"row_label_line": 264,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-FP8-dynamic/README.md",
"sha256": "8ae1b13343b1e329c54aa905fa6cf86b14b5c4a395893b5b8a1f7fe10a1d06e6",
"line": 265,
"raw_line": " 77.90 | ",
"row_label_line": 264,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic@ed35d7a:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-FP8-dynamic",
"artifact_revision": "ed35d7abe5d940da41b4ff06eb482feb0be8cb44",
"published_recovery_pct": 97.6
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "IFEval 0-shot prompt-level strict (without thinking)",
"full": 0.8996,
"quant_score": 0.8724,
"retention": 0.9697643397065362,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L169",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 169,
"raw_line": " 87.24 | ",
"row_label_line": 167,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 168,
"raw_line": " 89.96 | ",
"row_label_line": 167,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 97
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MMLU-Pro 0-shot custom-extract (without thinking)",
"full": 0.8347,
"quant_score": 0.8165,
"retention": 0.9781957589553133,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L188",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 188,
"raw_line": " 81.65 | ",
"row_label_line": 186,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 187,
"raw_line": " 83.47 | ",
"row_label_line": 186,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 97.8
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MATH-500 0-shot pass@1 (without thinking)",
"full": 0.848,
"quant_score": 0.876,
"retention": 1.0330188679245282,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L194",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 194,
"raw_line": " 87.60 | ",
"row_label_line": 192,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 193,
"raw_line": " 84.80 | ",
"row_label_line": 192,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 103.3
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (without thinking)",
"full": 0.8,
"quant_score": 0.6625,
"retention": 0.8281249999999999,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L200",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 200,
"raw_line": " 66.25 | ",
"row_label_line": 198,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 199,
"raw_line": " 80.00 | ",
"row_label_line": 198,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 82.8
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "GPQA Diamond 0-shot pass@1 (without thinking)",
"full": 0.732,
"quant_score": 0.7239,
"retention": 0.9889344262295082,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L206",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 206,
"raw_line": " 72.39 | ",
"row_label_line": 204,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 205,
"raw_line": " 73.20 | ",
"row_label_line": 204,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 98.9
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "LiveCodeBench v6 0-shot pass@1 (without thinking)",
"full": 0.7448,
"quant_score": 0.6819,
"retention": 0.9155477980665949,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L213",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 213,
"raw_line": " 68.19 | ",
"row_label_line": 211,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 212,
"raw_line": " 74.48 | ",
"row_label_line": 211,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:without thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 91.6
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "IFEval 0-shot prompt-level strict (with thinking)",
"full": 0.9439,
"quant_score": 0.9415,
"retention": 0.9974573577709503,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L236",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 236,
"raw_line": " 94.15 | ",
"row_label_line": 234,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 235,
"raw_line": " 94.39 | ",
"row_label_line": 234,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 99.7
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MMLU-Pro 0-shot custom-extract (with thinking)",
"full": 0.8519,
"quant_score": 0.8329,
"retention": 0.9776969127831906,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L255",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 255,
"raw_line": " 83.29 | ",
"row_label_line": 253,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 254,
"raw_line": " 85.19 | ",
"row_label_line": 253,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 97.8
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MATH-500 0-shot pass@1 (with thinking)",
"full": 0.8587,
"quant_score": 0.8507,
"retention": 0.990683591475486,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L261",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 261,
"raw_line": " 85.07 | ",
"row_label_line": 259,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 260,
"raw_line": " 85.87 | ",
"row_label_line": 259,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 99.1
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (with thinking)",
"full": 0.8875,
"quant_score": 0.8125,
"retention": 0.9154929577464789,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L267",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 267,
"raw_line": " 81.25 | ",
"row_label_line": 265,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 266,
"raw_line": " 88.75 | ",
"row_label_line": 265,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 91.5
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "GPQA Diamond 0-shot pass@1 (with thinking)",
"full": 0.8081,
"quant_score": 0.7761,
"retention": 0.9604009404776637,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L273",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 273,
"raw_line": " 77.61 | ",
"row_label_line": 271,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 272,
"raw_line": " 80.81 | ",
"row_label_line": 271,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 96
},
{
"model": "gemma-4-26B-A4B-it",
"model_id": "gemma-4-26b-a4b",
"params_b": 25.2,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "LiveCodeBench v6 0-shot pass@1 (with thinking)",
"full": 0.779,
"quant_score": 0.7276,
"retention": 0.934017971758665,
"baseline_quant": "BF16 (unquantized google/gemma-4-26B-A4B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-26B-A4B-it-NVFP4/blob/5557756b8dce33ac72f2bd702b11729fdba3b839/README.md#L280",
"source_id": "quant-redhatai-gemma-4-26b-a4b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 280,
"raw_line": " 72.76 | ",
"row_label_line": 278,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-26B-A4B-it-NVFP4/README.md",
"sha256": "c367d9809dc202f7416d6cc7efcb0dae72544e70b4668ea86e3141ca9639886d",
"line": 279,
"raw_line": " 77.90 | ",
"row_label_line": 278,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-26B-A4B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-26B-A4B-it-NVFP4@5557756:with thinking",
"artifact": "RedHatAI/gemma-4-26B-A4B-it-NVFP4",
"artifact_revision": "5557756b8dce33ac72f2bd702b11729fdba3b839",
"published_recovery_pct": 93.4
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "IFEval 0-shot prompt-level strict (without thinking)",
"full": 0.907,
"quant_score": 0.9125,
"retention": 1.00606394707828,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L160",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 160,
"raw_line": " 91.25 | ",
"row_label_line": 158,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 159,
"raw_line": " 90.70 | ",
"row_label_line": 158,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 100.6
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "MMLU-Pro 0-shot custom-extract (without thinking)",
"full": 0.8541,
"quant_score": 0.8544,
"retention": 1.0003512469265894,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L179",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 179,
"raw_line": " 85.44 | ",
"row_label_line": 177,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 178,
"raw_line": " 85.41 | ",
"row_label_line": 177,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 100
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "MATH-500 0-shot pass@1 (without thinking)",
"full": 0.894,
"quant_score": 0.8867,
"retention": 0.991834451901566,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L185",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 185,
"raw_line": " 88.67 | ",
"row_label_line": 183,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 184,
"raw_line": " 89.40 | ",
"row_label_line": 183,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 99.2
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (without thinking)",
"full": 0.6583,
"quant_score": 0.6833,
"retention": 1.0379766064104512,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L191",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 191,
"raw_line": " 68.33 | ",
"row_label_line": 189,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 190,
"raw_line": " 65.83 | ",
"row_label_line": 189,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 103.8
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "GPQA Diamond 0-shot pass@1 (without thinking)",
"full": 0.7744,
"quant_score": 0.7795,
"retention": 1.006585743801653,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L197",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 197,
"raw_line": " 77.95 | ",
"row_label_line": 195,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 196,
"raw_line": " 77.44 | ",
"row_label_line": 195,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 100.7
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "LiveCodeBench v6 0-shot pass@1 (without thinking)",
"full": 0.7143,
"quant_score": 0.7352,
"retention": 1.0292594148117036,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L204",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 204,
"raw_line": " 73.52 | ",
"row_label_line": 202,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 203,
"raw_line": " 71.43 | ",
"row_label_line": 202,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 102.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "IFEval 0-shot prompt-level strict (with thinking)",
"full": 0.9458,
"quant_score": 0.9421,
"retention": 0.996087967857898,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L227",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 227,
"raw_line": " 94.21 | ",
"row_label_line": 225,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 226,
"raw_line": " 94.58 | ",
"row_label_line": 225,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 99.6
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "MMLU-Pro 0-shot custom-extract (with thinking)",
"full": 0.8705,
"quant_score": 0.8701,
"retention": 0.9995404939689835,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L246",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 246,
"raw_line": " 87.01 | ",
"row_label_line": 244,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 245,
"raw_line": " 87.05 | ",
"row_label_line": 244,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 100
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "MATH-500 0-shot pass@1 (with thinking)",
"full": 0.8793,
"quant_score": 0.8807,
"retention": 1.0015921755942225,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L252",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 252,
"raw_line": " 88.07 | ",
"row_label_line": 250,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 251,
"raw_line": " 87.93 | ",
"row_label_line": 250,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 100.2
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (with thinking)",
"full": 0.8708,
"quant_score": 0.875,
"retention": 1.004823151125402,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L258",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 258,
"raw_line": " 87.50 | ",
"row_label_line": 256,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 257,
"raw_line": " 87.08 | ",
"row_label_line": 256,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 100.5
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 block-wise (data-free; llm-compressor model_free_ptq FP8_BLOCK)",
"metric": "GPQA Diamond 0-shot pass@1 (with thinking)",
"full": 0.8636,
"quant_score": 0.8687,
"retention": 1.0059055118110236,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-block/blob/d7242548c457ab4b45bd3adb8937f2659af4739d/README.md#L264",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 264,
"raw_line": " 86.87 | ",
"row_label_line": 262,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-block",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-block/README.md",
"sha256": "4c9e4e3440cd22cb702d5f638bb595ffe6437fbe4bd84cd7208390bb51c17c99",
"line": 263,
"raw_line": " 86.36 | ",
"row_label_line": 262,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-block@d724254:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-block",
"artifact_revision": "d7242548c457ab4b45bd3adb8937f2659af4739d",
"published_recovery_pct": 100.6
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "IFEval 0-shot prompt-level strict (without thinking)",
"full": 0.907,
"quant_score": 0.9107,
"retention": 1.0040793825799337,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L156",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 156,
"raw_line": " 91.07 | ",
"row_label_line": 154,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 155,
"raw_line": " 90.70 | ",
"row_label_line": 154,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 100.4
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MMLU-Pro 0-shot custom-extract (without thinking)",
"full": 0.8541,
"quant_score": 0.8532,
"retention": 0.9989462592202318,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L175",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 175,
"raw_line": " 85.32 | ",
"row_label_line": 173,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 174,
"raw_line": " 85.41 | ",
"row_label_line": 173,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 99.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MATH-500 0-shot pass@1 (without thinking)",
"full": 0.894,
"quant_score": 0.9027,
"retention": 1.009731543624161,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L181",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 181,
"raw_line": " 90.27 | ",
"row_label_line": 179,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 180,
"raw_line": " 89.40 | ",
"row_label_line": 179,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 101
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (without thinking)",
"full": 0.6583,
"quant_score": 0.6625,
"retention": 1.0063800698769558,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L187",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 187,
"raw_line": " 66.25 | ",
"row_label_line": 185,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 186,
"raw_line": " 65.83 | ",
"row_label_line": 185,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 100.6
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "GPQA Diamond 0-shot pass@1 (without thinking)",
"full": 0.7744,
"quant_score": 0.7811,
"retention": 1.0086518595041323,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L193",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 193,
"raw_line": " 78.11 | ",
"row_label_line": 191,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 192,
"raw_line": " 77.44 | ",
"row_label_line": 191,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 100.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "LiveCodeBench v6 0-shot pass@1 (without thinking)",
"full": 0.7143,
"quant_score": 0.7067,
"retention": 0.989360212795744,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L200",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 200,
"raw_line": " 70.67 | ",
"row_label_line": 198,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 199,
"raw_line": " 71.43 | ",
"row_label_line": 198,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 98.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "IFEval 0-shot prompt-level strict (with thinking)",
"full": 0.9458,
"quant_score": 0.9489,
"retention": 1.0032776485514907,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L223",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 223,
"raw_line": " 94.89 | ",
"row_label_line": 221,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 222,
"raw_line": " 94.58 | ",
"row_label_line": 221,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 100.3
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MMLU-Pro 0-shot custom-extract (with thinking)",
"full": 0.8705,
"quant_score": 0.8707,
"retention": 1.0002297530155082,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L242",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 242,
"raw_line": " 87.07 | ",
"row_label_line": 240,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 241,
"raw_line": " 87.05 | ",
"row_label_line": 240,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 100
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "MATH-500 0-shot pass@1 (with thinking)",
"full": 0.8793,
"quant_score": 0.8947,
"retention": 1.0175139315364492,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L248",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 248,
"raw_line": " 89.47 | ",
"row_label_line": 246,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 247,
"raw_line": " 87.93 | ",
"row_label_line": 246,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 101.7
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (with thinking)",
"full": 0.8708,
"quant_score": 0.9,
"retention": 1.033532384014699,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L254",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 254,
"raw_line": " 90.00 | ",
"row_label_line": 252,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 253,
"raw_line": " 87.08 | ",
"row_label_line": 252,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 103.4
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "GPQA Diamond 0-shot pass@1 (with thinking)",
"full": 0.8636,
"quant_score": 0.8603,
"retention": 0.9961787864752201,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L260",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 260,
"raw_line": " 86.03 | ",
"row_label_line": 258,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 259,
"raw_line": " 86.36 | ",
"row_label_line": 258,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 99.6
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "FP8 dynamic (data-free; FP8 weights per-channel, activations dynamic per-token; llm-compressor model_free_ptq FP8_DYNAMIC)",
"metric": "LiveCodeBench v6 0-shot pass@1 (with thinking)",
"full": 0.7429,
"quant_score": 0.7924,
"retention": 1.0666307713016554,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-FP8-dynamic/blob/d4ab4f579dd3516f97d8a6a4c98d0653480bad15/README.md#L267",
"source_id": "quant-redhatai-gemma-4-31b-it-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 267,
"raw_line": " 79.24 | ",
"row_label_line": 265,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-FP8-dynamic/README.md",
"sha256": "61792c089e54f8ff500567b0ce1a6bafb499546319ef3c3d176c4db091aee6c9",
"line": 266,
"raw_line": " 74.29 | ",
"row_label_line": 265,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-FP8-dynamic@d4ab4f5:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-FP8-dynamic",
"artifact_revision": "d4ab4f579dd3516f97d8a6a4c98d0653480bad15",
"published_recovery_pct": 106.7
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "IFEval 0-shot prompt-level strict (without thinking)",
"full": 0.907,
"quant_score": 0.8977,
"retention": 0.9897464167585446,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L168",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 168,
"raw_line": " 89.77 | ",
"row_label_line": 166,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 167,
"raw_line": " 90.70 | ",
"row_label_line": 166,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 99
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MMLU-Pro 0-shot custom-extract (without thinking)",
"full": 0.8541,
"quant_score": 0.845,
"retention": 0.9893455098934552,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L187",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 187,
"raw_line": " 84.50 | ",
"row_label_line": 185,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 186,
"raw_line": " 85.41 | ",
"row_label_line": 185,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 98.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MATH-500 0-shot pass@1 (without thinking)",
"full": 0.894,
"quant_score": 0.8507,
"retention": 0.951565995525727,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L193",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 193,
"raw_line": " 85.07 | ",
"row_label_line": 191,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 192,
"raw_line": " 89.40 | ",
"row_label_line": 191,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 95.2
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (without thinking)",
"full": 0.6583,
"quant_score": 0.65,
"retention": 0.9873917666717302,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L199",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 199,
"raw_line": " 65.00 | ",
"row_label_line": 197,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 198,
"raw_line": " 65.83 | ",
"row_label_line": 197,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 98.7
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "GPQA Diamond 0-shot pass@1 (without thinking)",
"full": 0.7744,
"quant_score": 0.766,
"retention": 0.9891528925619834,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L205",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 205,
"raw_line": " 76.60 | ",
"row_label_line": 203,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 204,
"raw_line": " 77.44 | ",
"row_label_line": 203,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 98.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "LiveCodeBench v6 0-shot pass@1 (without thinking)",
"full": 0.7143,
"quant_score": 0.7067,
"retention": 0.989360212795744,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L212",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 212,
"raw_line": " 70.67 | ",
"row_label_line": 210,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 211,
"raw_line": " 71.43 | ",
"row_label_line": 210,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:without thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 98.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "IFEval 0-shot prompt-level strict (with thinking)",
"full": 0.9458,
"quant_score": 0.9396,
"retention": 0.9934447028970184,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L235",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 235,
"raw_line": " 93.96 | ",
"row_label_line": 233,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Instruction Following"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 234,
"raw_line": " 94.58 | ",
"row_label_line": 233,
"row_label": "IFEval (0-shot, prompt-level strict)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 99.3
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MMLU-Pro 0-shot custom-extract (with thinking)",
"full": 0.8705,
"quant_score": 0.8638,
"retention": 0.9923032739804709,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L254",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 254,
"raw_line": " 86.38 | ",
"row_label_line": 252,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 253,
"raw_line": " 87.05 | ",
"row_label_line": 252,
"row_label": "MMLU-Pro (0-shot, custom-extract)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 99.2
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "MATH-500 0-shot pass@1 (with thinking)",
"full": 0.8793,
"quant_score": 0.87,
"retention": 0.9894234049812349,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L260",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 260,
"raw_line": " 87.00 | ",
"row_label_line": 258,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 259,
"raw_line": " 87.93 | ",
"row_label_line": 258,
"row_label": "MATH-500 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 98.9
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "AIME 2025 0-shot pass@1, mean of 8 seeds (with thinking)",
"full": 0.8708,
"quant_score": 0.8625,
"retention": 0.9904685346807534,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L266",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 266,
"raw_line": " 86.25 | ",
"row_label_line": 264,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 265,
"raw_line": " 87.08 | ",
"row_label_line": 264,
"row_label": "AIME 2025 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 99
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "GPQA Diamond 0-shot pass@1 (with thinking)",
"full": 0.8636,
"quant_score": 0.8704,
"retention": 1.0078740157480315,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L272",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 272,
"raw_line": " 87.04 | ",
"row_label_line": 270,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Reasoning"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 271,
"raw_line": " 86.36 | ",
"row_label_line": 270,
"row_label": "GPQA Diamond (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 100.8
},
{
"model": "gemma-4-31B-it",
"model_id": "gemma-4-31b",
"params_b": 30.7,
"quant": "NVFP4 (FP4 weights/activations with per-group scaling; llm-compressor NVFP4)",
"metric": "LiveCodeBench v6 0-shot pass@1 (with thinking)",
"full": 0.7429,
"quant_score": 0.779,
"retention": 1.0485933503836316,
"baseline_quant": "BF16 (unquantized google/gemma-4-31B-it, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/gemma-4-31B-it-NVFP4/blob/5df75ccad9b74546f5be758a2a68acc2e16bbc9b/README.md#L279",
"source_id": "quant-redhatai-gemma-4-31b-it-nvfp4",
"evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 279,
"raw_line": " 77.90 | ",
"row_label_line": 277,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "RedHatAI/gemma-4-31B-it-NVFP4",
"category": "Coding"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__gemma-4-31B-it-NVFP4/README.md",
"sha256": "353739bf71e2688b9ac6c26bd3914676543699d17f6ccb345c9373094c01353b",
"line": 278,
"raw_line": " 74.29 | ",
"row_label_line": 277,
"row_label": "LiveCodeBench v6 (0-shot, pass@1)",
"column_header": "google/gemma-4-31B-it"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat evaluation served with vLLM (OpenAI-compatible API) using the neuralmagic forks of lm-evaluation-harness and lighteval; 3 seeds averaged (AIME 2025: 8 seeds). Gemma 4 is Apache-2.0; the pinned repository declares license: apache-2.0. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/gemma-4-31B-it-NVFP4@5df75cc:with thinking",
"artifact": "RedHatAI/gemma-4-31B-it-NVFP4",
"artifact_revision": "5df75ccad9b74546f5be758a2a68acc2e16bbc9b",
"published_recovery_pct": 104.9
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 block-wise (llm-compressor FP8_BLOCK)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7662,
"quant_score": 0.7662,
"retention": 1,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-block/blob/d65f13db6dee2495b3c8e32339cca3e616534e23/README.md#L281",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 281,
"raw_line": " 76.62 (100.00%) | ",
"row_label_line": 278,
"row_label": "Average Score",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-block",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 279,
"raw_line": " 76.62 | ",
"row_label_line": 278,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/granite-4.0-h-small-FP8-block@d65f13d:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-block",
"artifact_revision": "d65f13db6dee2495b3c8e32339cca3e616534e23",
"published_recovery_pct": 100
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 block-wise (llm-compressor FP8_BLOCK)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8522,
"quant_score": 0.8552,
"retention": 1.0035203003989674,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-block/blob/d65f13db6dee2495b3c8e32339cca3e616534e23/README.md#L246",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 246,
"raw_line": " 85.52 (100.36%) | ",
"row_label_line": 243,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-block",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 244,
"raw_line": " 85.22 | ",
"row_label_line": 243,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-block@d65f13d:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-block",
"artifact_revision": "d65f13db6dee2495b3c8e32339cca3e616534e23",
"published_recovery_pct": 100.36
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 block-wise (llm-compressor FP8_BLOCK)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7715,
"quant_score": 0.7723,
"retention": 1.0010369410239792,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-block/blob/d65f13db6dee2495b3c8e32339cca3e616534e23/README.md#L260",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 260,
"raw_line": " 77.23 (100.09%) | ",
"row_label_line": 257,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-block",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 258,
"raw_line": " 77.15 | ",
"row_label_line": 257,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-block@d65f13d:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-block",
"artifact_revision": "d65f13db6dee2495b3c8e32339cca3e616534e23",
"published_recovery_pct": 100.09
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 block-wise (llm-compressor FP8_BLOCK)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.542,
"quant_score": 0.5332,
"retention": 0.9837638376383764,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-block/blob/d65f13db6dee2495b3c8e32339cca3e616534e23/README.md#L332",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 332,
"raw_line": " 53.32 (98.38%) | ",
"row_label_line": 329,
"row_label": "Average Score",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-block",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 330,
"raw_line": " 54.20 | ",
"row_label_line": 329,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/granite-4.0-h-small-FP8-block@d65f13d:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-block",
"artifact_revision": "d65f13db6dee2495b3c8e32339cca3e616534e23",
"published_recovery_pct": 98.38
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 block-wise (llm-compressor FP8_BLOCK)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config)",
"full": 0.8753,
"quant_score": 0.8669,
"retention": 0.9904032903004685,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-block/blob/d65f13db6dee2495b3c8e32339cca3e616534e23/README.md#L290",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 290,
"raw_line": " 86.69 (99.04%) | ",
"row_label_line": 287,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-block",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 288,
"raw_line": " 87.53 | ",
"row_label_line": 287,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-block@d65f13d:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-block",
"artifact_revision": "d65f13db6dee2495b3c8e32339cca3e616534e23",
"published_recovery_pct": 99.04
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 block-wise (llm-compressor FP8_BLOCK)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published)",
"full": 0.4799,
"quant_score": 0.4793,
"retention": 0.9987497395290685,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-block/blob/d65f13db6dee2495b3c8e32339cca3e616534e23/README.md#L325",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-block",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 325,
"raw_line": " 47.93 (99.88%) | ",
"row_label_line": 322,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-block",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-block/README.md",
"sha256": "b304319c3b1276101a61d8110ec589c582dd5fdec3fd4fa3fe70eadcdee87f40",
"line": 323,
"raw_line": " 47.99 | ",
"row_label_line": 322,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-block@d65f13d:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-block",
"artifact_revision": "d65f13db6dee2495b3c8e32339cca3e616534e23",
"published_recovery_pct": 99.88
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7662,
"quant_score": 0.7651,
"retention": 0.998564343513443,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L299",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 299,
"raw_line": " 76.51 (99.86%) | ",
"row_label_line": 295,
"row_label": "Average Score",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 296,
"raw_line": " 76.62 | ",
"row_label_line": 295,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"artifact_revision": "440b960193f3e871ebf74f942e630fbbd5012602",
"published_recovery_pct": 99.86
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8522,
"quant_score": 0.8484,
"retention": 0.9955409528279747,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L264",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 264,
"raw_line": " 84.84 (99.56%) | ",
"row_label_line": 260,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 261,
"raw_line": " 85.22 | ",
"row_label_line": 260,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"artifact_revision": "440b960193f3e871ebf74f942e630fbbd5012602",
"published_recovery_pct": 99.56
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7715,
"quant_score": 0.7718,
"retention": 1.0003888528839922,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L278",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 278,
"raw_line": " 77.18 (100.03%) | ",
"row_label_line": 274,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 275,
"raw_line": " 77.15 | ",
"row_label_line": 274,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"artifact_revision": "440b960193f3e871ebf74f942e630fbbd5012602",
"published_recovery_pct": 100.03
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.542,
"quant_score": 0.5299,
"retention": 0.9776752767527676,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L350",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 350,
"raw_line": " 52.99 (97.77%) | ",
"row_label_line": 346,
"row_label": "Average Score",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 347,
"raw_line": " 54.20 | ",
"row_label_line": 346,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"artifact_revision": "440b960193f3e871ebf74f942e630fbbd5012602",
"published_recovery_pct": 97.77
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config)",
"full": 0.8753,
"quant_score": 0.8741,
"retention": 0.9986290414714956,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L308",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 308,
"raw_line": " 87.41 (99.86%) | ",
"row_label_line": 304,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 305,
"raw_line": " 87.53 | ",
"row_label_line": 304,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"artifact_revision": "440b960193f3e871ebf74f942e630fbbd5012602",
"published_recovery_pct": 99.86
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published)",
"full": 0.4799,
"quant_score": 0.4758,
"retention": 0.9914565534486351,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L343",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 343,
"raw_line": " 47.58 (99.15%) | ",
"row_label_line": 339,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 340,
"raw_line": " 47.99 | ",
"row_label_line": 339,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Four-column comparison table; this row uses the column for this repository's own artifact. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-small-FP8-dynamic",
"artifact_revision": "440b960193f3e871ebf74f942e630fbbd5012602",
"published_recovery_pct": 99.15
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 (IBM official ibm-granite/granite-4.0-h-small-FP8 checkpoint; recipe not described in this card)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.7662,
"quant_score": 0.7659,
"retention": 0.9996084573218481,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L297",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 297,
"raw_line": " 76.59 (99.96%) | ",
"row_label_line": 295,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small-FP8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 296,
"raw_line": " 76.62 | ",
"row_label_line": 295,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat measured IBM's own FP8 release in the same table; extracted once, from this card only (the FP8-block card repeats the identical table). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v1",
"artifact": "ibm-granite/granite-4.0-h-small-FP8",
"artifact_revision": null,
"published_recovery_pct": 99.96
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 (IBM official ibm-granite/granite-4.0-h-small-FP8 checkpoint; recipe not described in this card)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8522,
"quant_score": 0.8529,
"retention": 1.0008214034264258,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L262",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 262,
"raw_line": " 85.29 (100.09%) | ",
"row_label_line": 260,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small-FP8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 261,
"raw_line": " 85.22 | ",
"row_label_line": 260,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat measured IBM's own FP8 release in the same table; extracted once, from this card only (the FP8-block card repeats the identical table). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v1",
"artifact": "ibm-granite/granite-4.0-h-small-FP8",
"artifact_revision": null,
"published_recovery_pct": 100.09
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 (IBM official ibm-granite/granite-4.0-h-small-FP8 checkpoint; recipe not described in this card)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.7715,
"quant_score": 0.7718,
"retention": 1.0003888528839922,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L276",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 276,
"raw_line": " 77.18 (100.03%) | ",
"row_label_line": 274,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small-FP8",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 275,
"raw_line": " 77.15 | ",
"row_label_line": 274,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat measured IBM's own FP8 release in the same table; extracted once, from this card only (the FP8-block card repeats the identical table). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v1",
"artifact": "ibm-granite/granite-4.0-h-small-FP8",
"artifact_revision": null,
"published_recovery_pct": 100.03
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 (IBM official ibm-granite/granite-4.0-h-small-FP8 checkpoint; recipe not described in this card)",
"metric": "OpenLLM v2 average of 6 tasks (IFEval, BBH, MATH-Hard, GPQA, MuSR, MMLU-Pro; lm-evaluation-harness `leaderboard`, chat template, vLLM; values exactly as published, per-task normalisation not stated)",
"full": 0.542,
"quant_score": 0.5356,
"retention": 0.9881918819188192,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L348",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 348,
"raw_line": " 53.56 (98.82%) | ",
"row_label_line": 346,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small-FP8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 347,
"raw_line": " 54.20 | ",
"row_label_line": 346,
"row_label": "Average Score",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat measured IBM's own FP8 release in the same table; extracted once, from this card only (the FP8-block card repeats the identical table). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v2",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v2",
"artifact": "ibm-granite/granite-4.0-h-small-FP8",
"artifact_revision": null,
"published_recovery_pct": 98.82
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 (IBM official ibm-granite/granite-4.0-h-small-FP8 checkpoint; recipe not described in this card)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config)",
"full": 0.8753,
"quant_score": 0.8717,
"retention": 0.9958871244144866,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L306",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 306,
"raw_line": " 87.17 (99.59%) | ",
"row_label_line": 304,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "ibm-granite/granite-4.0-h-small-FP8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 305,
"raw_line": " 87.53 | ",
"row_label_line": 304,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat measured IBM's own FP8 release in the same table; extracted once, from this card only (the FP8-block card repeats the identical table). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v2",
"artifact": "ibm-granite/granite-4.0-h-small-FP8",
"artifact_revision": null,
"published_recovery_pct": 99.59
},
{
"model": "granite-4.0-h-small",
"model_id": "granite-4-0-h-small",
"params_b": 32.2,
"quant": "FP8 (IBM official ibm-granite/granite-4.0-h-small-FP8 checkpoint; recipe not described in this card)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config; value as published)",
"full": 0.4799,
"quant_score": 0.4763,
"retention": 0.9924984371744113,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-small, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-small-FP8-dynamic/blob/440b960193f3e871ebf74f942e630fbbd5012602/README.md#L341",
"source_id": "quant-redhatai-granite-4-0-h-small-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 341,
"raw_line": " 47.63 (99.26%) | ",
"row_label_line": 339,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small-FP8",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-small-FP8-dynamic/README.md",
"sha256": "f7108b7d6e7ccfddddc4764deb46a987e74c4cfd46d98dc9212f03235735829f",
"line": 340,
"raw_line": " 47.99 | ",
"row_label_line": 339,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-small"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Red Hat measured IBM's own FP8 release in the same table; extracted once, from this card only (the FP8-block card repeats the identical table). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-small-FP8-dynamic@440b960:openllm-v2",
"artifact": "ibm-granite/granite-4.0-h-small-FP8",
"artifact_revision": null,
"published_recovery_pct": 99.26
},
{
"model": "granite-4.0-h-tiny",
"model_id": null,
"params_b": 7,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.6692,
"quant_score": 0.6675,
"retention": 0.9974596533173938,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-tiny, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-tiny-FP8-dynamic/blob/85b19401d507a7d3ddfd93065dd5adf68ecc46a1/README.md#L246",
"source_id": "quant-redhatai-granite-4-0-h-tiny-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 246,
"raw_line": "| | **Average** | **66.92** | **66.75** | **99.73** |",
"row_label_line": 246,
"row_label": "Average",
"column_header": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 246,
"raw_line": "| | **Average** | **66.92** | **66.75** | **99.73** |",
"row_label_line": 246,
"row_label": "Average",
"column_header": "ibm-granite/granite-4.0-h-tiny"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. granite-4.0-h-tiny is not in data/models.json (model_id null); params_b from the card description (7B). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic@85b1940:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"artifact_revision": "85b19401d507a7d3ddfd93065dd5adf68ecc46a1",
"published_recovery_pct": 99.73
},
{
"model": "granite-4.0-h-tiny",
"model_id": null,
"params_b": 7,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.8044,
"quant_score": 0.7983,
"retention": 0.9924167081054202,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-tiny, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-tiny-FP8-dynamic/blob/85b19401d507a7d3ddfd93065dd5adf68ecc46a1/README.md#L241",
"source_id": "quant-redhatai-granite-4-0-h-tiny-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 241,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 80.44 | 79.83 | 99.24 |",
"row_label_line": 241,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 241,
"raw_line": "| | GSM8K (Strict-Match, 5-shot) | 80.44 | 79.83 | 99.24 |",
"row_label_line": 241,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-tiny"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. granite-4.0-h-tiny is not in data/models.json (model_id null); params_b from the card description (7B). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic@85b1940:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"artifact_revision": "85b19401d507a7d3ddfd93065dd5adf68ecc46a1",
"published_recovery_pct": 99.24
},
{
"model": "granite-4.0-h-tiny",
"model_id": null,
"params_b": 7,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.6646,
"quant_score": 0.6633,
"retention": 0.998043936202227,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-tiny, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-tiny-FP8-dynamic/blob/85b19401d507a7d3ddfd93065dd5adf68ecc46a1/README.md#L243",
"source_id": "quant-redhatai-granite-4-0-h-tiny-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 243,
"raw_line": "| | MMLU (Acc, 5-shot) | 66.46 | 66.33 | 99.80 |",
"row_label_line": 243,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 243,
"raw_line": "| | MMLU (Acc, 5-shot) | 66.46 | 66.33 | 99.80 |",
"row_label_line": 243,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-tiny"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. granite-4.0-h-tiny is not in data/models.json (model_id null); params_b from the card description (7B). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic@85b1940:openllm-v1",
"artifact": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"artifact_revision": "85b19401d507a7d3ddfd93065dd5adf68ecc46a1",
"published_recovery_pct": 99.8
},
{
"model": "granite-4.0-h-tiny",
"model_id": null,
"params_b": 7,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "OpenLLM v2 subset average of 2 tasks (IFEval inst-level strict, MMLU-Pro 5-shot) as published",
"full": 0.5843,
"quant_score": 0.5858,
"retention": 1.002567174396714,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-tiny, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-tiny-FP8-dynamic/blob/85b19401d507a7d3ddfd93065dd5adf68ecc46a1/README.md#L249",
"source_id": "quant-redhatai-granite-4-0-h-tiny-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 249,
"raw_line": "| | **Average** | **58.43** | **58.58** | **100.13** |",
"row_label_line": 249,
"row_label": "Average",
"column_header": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 249,
"raw_line": "| | **Average** | **58.43** | **58.58** | **100.13** |",
"row_label_line": 249,
"row_label": "Average",
"column_header": "ibm-granite/granite-4.0-h-tiny"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. granite-4.0-h-tiny is not in data/models.json (model_id null); params_b from the card description (7B). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 2 tasks: IFEval, MMLU-Pro",
"study": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic@85b1940:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"artifact_revision": "85b19401d507a7d3ddfd93065dd5adf68ecc46a1",
"published_recovery_pct": 100.13
},
{
"model": "granite-4.0-h-tiny",
"model_id": null,
"params_b": 7,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "IFEval 0-shot instruction-level strict acc (OpenLLM v2 config)",
"full": 0.7062,
"quant_score": 0.711,
"retention": 1.0067969413763804,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-tiny, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-tiny-FP8-dynamic/blob/85b19401d507a7d3ddfd93065dd5adf68ecc46a1/README.md#L247",
"source_id": "quant-redhatai-granite-4-0-h-tiny-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 247,
"raw_line": "| **OpenLLM V2** | IFEval (Inst Level Strict Acc, 0-shot) | 70.62 | 71.10 | 100.68 |",
"row_label_line": 247,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 247,
"raw_line": "| **OpenLLM V2** | IFEval (Inst Level Strict Acc, 0-shot) | 70.62 | 71.10 | 100.68 |",
"row_label_line": 247,
"row_label": "IFEval (Inst Level Strict Acc, 0-shot)",
"column_header": "ibm-granite/granite-4.0-h-tiny"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. granite-4.0-h-tiny is not in data/models.json (model_id null); params_b from the card description (7B). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic@85b1940:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"artifact_revision": "85b19401d507a7d3ddfd93065dd5adf68ecc46a1",
"published_recovery_pct": 100.68
},
{
"model": "granite-4.0-h-tiny",
"model_id": null,
"params_b": 7,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC)",
"metric": "MMLU-Pro 5-shot (OpenLLM v2 config)",
"full": 0.4624,
"quant_score": 0.4605,
"retention": 0.9958910034602074,
"baseline_quant": "BF16 (unquantized ibm-granite/granite-4.0-h-tiny, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/granite-4.0-h-tiny-FP8-dynamic/blob/85b19401d507a7d3ddfd93065dd5adf68ecc46a1/README.md#L248",
"source_id": "quant-redhatai-granite-4-0-h-tiny-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 248,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 46.24 | 46.05 | 99.59 |",
"row_label_line": 248,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"category": "OpenLLM V2"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__granite-4.0-h-tiny-FP8-dynamic/README.md",
"sha256": "e08146ee16d933ea9703e60e7238b7c93760c1c2144ff37f88a8a006fd8ea5ac",
"line": 248,
"raw_line": "| | MMLU-Pro (Acc, 5-shot) | 46.24 | 46.05 | 99.59 |",
"row_label_line": 248,
"row_label": "MMLU-Pro (Acc, 5-shot)",
"column_header": "ibm-granite/granite-4.0-h-tiny"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. granite-4.0-h-tiny is not in data/models.json (model_id null); params_b from the card description (7B). Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic@85b1940:openllm-v2",
"artifact": "RedHatAI/granite-4.0-h-tiny-FP8-dynamic",
"artifact_revision": "85b19401d507a7d3ddfd93065dd5adf68ecc46a1",
"published_recovery_pct": 99.59
},
{
"model": "Apertus-70B-Instruct-2509",
"model_id": null,
"params_b": 70,
"quant": "FP8 dynamic (llm-compressor FP8_dynamic)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.735,
"quant_score": 0.731,
"retention": 0.9945578231292517,
"baseline_quant": "BF16 (unquantized swiss-ai/Apertus-70B-Instruct-2509, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic/blob/79a917d6a8034910fa610b097dc61e96eac65d29/README.md#L192",
"source_id": "quant-redhatai-apertus-70b-instruct-2509-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Apertus-70B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "fc8d745fa1e00538ecd789ccd9c3169ae9f79e806164d7a275e0a52365823440",
"line": 192,
"raw_line": " 73.10 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Apertus-70B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "fc8d745fa1e00538ecd789ccd9c3169ae9f79e806164d7a275e0a52365823440",
"line": 191,
"raw_line": " 73.50 | ",
"row_label_line": 190,
"row_label": "Average Score",
"column_header": "swiss-ai/Apertus-70B-Instruct-2509"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Apertus is not in data/models.json (model_id null); params_b from the model name. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic@79a917d:openllm-v1",
"artifact": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic",
"artifact_revision": "79a917d6a8034910fa610b097dc61e96eac65d29",
"published_recovery_pct": 99.5
},
{
"model": "Apertus-70B-Instruct-2509",
"model_id": null,
"params_b": 70,
"quant": "FP8 dynamic (llm-compressor FP8_dynamic)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.7369,
"quant_score": 0.7309,
"retention": 0.9918577826027954,
"baseline_quant": "BF16 (unquantized swiss-ai/Apertus-70B-Instruct-2509, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic/blob/79a917d6a8034910fa610b097dc61e96eac65d29/README.md#L162",
"source_id": "quant-redhatai-apertus-70b-instruct-2509-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Apertus-70B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "fc8d745fa1e00538ecd789ccd9c3169ae9f79e806164d7a275e0a52365823440",
"line": 162,
"raw_line": " 73.09 | ",
"row_label_line": 160,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Apertus-70B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "fc8d745fa1e00538ecd789ccd9c3169ae9f79e806164d7a275e0a52365823440",
"line": 161,
"raw_line": " 73.69 | ",
"row_label_line": 160,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "swiss-ai/Apertus-70B-Instruct-2509"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Apertus is not in data/models.json (model_id null); params_b from the model name. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic@79a917d:openllm-v1",
"artifact": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic",
"artifact_revision": "79a917d6a8034910fa610b097dc61e96eac65d29",
"published_recovery_pct": 99.2
},
{
"model": "Apertus-70B-Instruct-2509",
"model_id": null,
"params_b": 70,
"quant": "FP8 dynamic (llm-compressor FP8_dynamic)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.6921,
"quant_score": 0.6825,
"retention": 0.9861291720849589,
"baseline_quant": "BF16 (unquantized swiss-ai/Apertus-70B-Instruct-2509, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic/blob/79a917d6a8034910fa610b097dc61e96eac65d29/README.md#L174",
"source_id": "quant-redhatai-apertus-70b-instruct-2509-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Apertus-70B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "fc8d745fa1e00538ecd789ccd9c3169ae9f79e806164d7a275e0a52365823440",
"line": 174,
"raw_line": " 68.25 | ",
"row_label_line": 172,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Apertus-70B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "fc8d745fa1e00538ecd789ccd9c3169ae9f79e806164d7a275e0a52365823440",
"line": 173,
"raw_line": " 69.21 | ",
"row_label_line": 172,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "swiss-ai/Apertus-70B-Instruct-2509"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Apertus is not in data/models.json (model_id null); params_b from the model name. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic@79a917d:openllm-v1",
"artifact": "RedHatAI/Apertus-70B-Instruct-2509-FP8-dynamic",
"artifact_revision": "79a917d6a8034910fa610b097dc61e96eac65d29",
"published_recovery_pct": 98.6
},
{
"model": "Apertus-8B-Instruct-2509",
"model_id": null,
"params_b": 8,
"quant": "FP8 dynamic (llm-compressor FP8_dynamic)",
"metric": "OpenLLM v1 average of 6 tasks (ARC-Challenge 25-shot, GSM8K 5-shot, HellaSwag 10-shot, MMLU 5-shot, TruthfulQA MC2 0-shot, Winogrande 5-shot; lm-evaluation-harness `openllm`, vLLM)",
"full": 0.6654,
"quant_score": 0.6633,
"retention": 0.9968440036068529,
"baseline_quant": "BF16 (unquantized swiss-ai/Apertus-8B-Instruct-2509, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic/blob/e9b17a517caa151279eeaf05fc973b3a11d9e0f9/README.md#L350",
"source_id": "quant-redhatai-apertus-8b-instruct-2509-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Apertus-8B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "1463f5c6bb64cd2c12061d8319898f260400c92271c6b3c13e47e44009e878e0",
"line": 350,
"raw_line": " 66.33 | ",
"row_label_line": 348,
"row_label": "Average Score",
"column_header": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Apertus-8B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "1463f5c6bb64cd2c12061d8319898f260400c92271c6b3c13e47e44009e878e0",
"line": 349,
"raw_line": " 66.54 | ",
"row_label_line": 348,
"row_label": "Average Score",
"column_header": "swiss-ai/Apertus-8B-Instruct-2509"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Apertus is not in data/models.json (model_id null); params_b from the model name. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "suite average of 6 tasks: OpenLLM v1",
"study": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic@e9b17a5:openllm-v1",
"artifact": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic",
"artifact_revision": "e9b17a517caa151279eeaf05fc973b3a11d9e0f9",
"published_recovery_pct": 99.7
},
{
"model": "Apertus-8B-Instruct-2509",
"model_id": null,
"params_b": 8,
"quant": "FP8 dynamic (llm-compressor FP8_dynamic)",
"metric": "GSM8K 5-shot strict-match (lm-evaluation-harness, vLLM)",
"full": 0.5807,
"quant_score": 0.555,
"retention": 0.9557430687101774,
"baseline_quant": "BF16 (unquantized swiss-ai/Apertus-8B-Instruct-2509, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic/blob/e9b17a517caa151279eeaf05fc973b3a11d9e0f9/README.md#L320",
"source_id": "quant-redhatai-apertus-8b-instruct-2509-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Apertus-8B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "1463f5c6bb64cd2c12061d8319898f260400c92271c6b3c13e47e44009e878e0",
"line": 320,
"raw_line": " 55.50 | ",
"row_label_line": 318,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Apertus-8B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "1463f5c6bb64cd2c12061d8319898f260400c92271c6b3c13e47e44009e878e0",
"line": 319,
"raw_line": " 58.07 | ",
"row_label_line": 318,
"row_label": "GSM8K (Strict-Match, 5-shot)",
"column_header": "swiss-ai/Apertus-8B-Instruct-2509"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Apertus is not in data/models.json (model_id null); params_b from the model name. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic@e9b17a5:openllm-v1",
"artifact": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic",
"artifact_revision": "e9b17a517caa151279eeaf05fc973b3a11d9e0f9",
"published_recovery_pct": 95.6
},
{
"model": "Apertus-8B-Instruct-2509",
"model_id": null,
"params_b": 8,
"quant": "FP8 dynamic (llm-compressor FP8_dynamic)",
"metric": "MMLU 5-shot acc (lm-evaluation-harness, vLLM)",
"full": 0.6197,
"quant_score": 0.6186,
"retention": 0.9982249475552687,
"baseline_quant": "BF16 (unquantized swiss-ai/Apertus-8B-Instruct-2509, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic/blob/e9b17a517caa151279eeaf05fc973b3a11d9e0f9/README.md#L332",
"source_id": "quant-redhatai-apertus-8b-instruct-2509-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Apertus-8B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "1463f5c6bb64cd2c12061d8319898f260400c92271c6b3c13e47e44009e878e0",
"line": 332,
"raw_line": " 61.86 | ",
"row_label_line": 330,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic",
"category": "OpenLLM V1"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Apertus-8B-Instruct-2509-FP8-dynamic/README.md",
"sha256": "1463f5c6bb64cd2c12061d8319898f260400c92271c6b3c13e47e44009e878e0",
"line": 331,
"raw_line": " 61.97 | ",
"row_label_line": 330,
"row_label": "MMLU (Acc, 5-shot)",
"column_header": "swiss-ai/Apertus-8B-Instruct-2509"
},
"transformation": "Published percentages / 100; retention = quant_score / baseline_score recomputed (unrounded), not copied from the Recovery column (published recovery checked within 0.5 points)",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Apertus is not in data/models.json (model_id null); params_b from the model name. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic@e9b17a5:openllm-v1",
"artifact": "RedHatAI/Apertus-8B-Instruct-2509-FP8-dynamic",
"artifact_revision": "e9b17a517caa151279eeaf05fc973b3a11d9e0f9",
"published_recovery_pct": 99.8
},
{
"model": "whisper-large-v3",
"model_id": "whisper-large-v3",
"params_b": 1.55,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC; table column header says \"w8a8\")",
"metric": "LibriSpeech test-clean WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.021725,
"quant_score": 0.02097,
"retention": 1.036003814973772,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-FP8-dynamic/blob/a972f9c7c573f57054608ca4c47594f68da64031/README.md#L182",
"source_id": "quant-redhatai-whisper-large-v3-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-FP8-dynamic/README.md",
"sha256": "e04d9543a13d817998f555ea5e8c0dae1c9e0c8049d4c9b3b2f1d270ce0943da",
"line": 182,
"raw_line": " 2.097 | ",
"row_label_line": 180,
"row_label": "test-clean",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-FP8-dynamic/README.md",
"sha256": "e04d9543a13d817998f555ea5e8c0dae1c9e0c8049d4c9b3b2f1d270ce0943da",
"line": 181,
"raw_line": " 2.1725 | ",
"row_label_line": 180,
"row_label": "test-clean",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-FP8-dynamic@a972f9c:librispeech",
"artifact": "RedHatAI/whisper-large-v3-FP8-dynamic",
"artifact_revision": "a972f9c7c573f57054608ca4c47594f68da64031",
"direction": "lower-is-better",
"published_recovery_pct": 103.6
},
{
"model": "whisper-large-v3",
"model_id": "whisper-large-v3",
"params_b": 1.55,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC; table column header says \"w8a8\")",
"metric": "LibriSpeech test-other WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.03903,
"quant_score": 0.039617,
"retention": 0.9851831284549563,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-FP8-dynamic/blob/a972f9c7c573f57054608ca4c47594f68da64031/README.md#L188",
"source_id": "quant-redhatai-whisper-large-v3-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-FP8-dynamic/README.md",
"sha256": "e04d9543a13d817998f555ea5e8c0dae1c9e0c8049d4c9b3b2f1d270ce0943da",
"line": 188,
"raw_line": " 3.9617 | ",
"row_label_line": 186,
"row_label": "test-other",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-FP8-dynamic/README.md",
"sha256": "e04d9543a13d817998f555ea5e8c0dae1c9e0c8049d4c9b3b2f1d270ce0943da",
"line": 187,
"raw_line": " 3.903 | ",
"row_label_line": 186,
"row_label": "test-other",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-FP8-dynamic@a972f9c:librispeech",
"artifact": "RedHatAI/whisper-large-v3-FP8-dynamic",
"artifact_revision": "a972f9c7c573f57054608ca4c47594f68da64031",
"direction": "lower-is-better",
"published_recovery_pct": 98.52
},
{
"model": "whisper-large-v3",
"model_id": "whisper-large-v3",
"params_b": 1.55,
"quant": "INT8 W8A8 (SmoothQuant + GPTQ via llm-compressor)",
"metric": "LibriSpeech test-clean WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.021725,
"quant_score": 0.020744,
"retention": 1.0472907828769764,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-quantized.w8a8/blob/11d614fb6fc82faeaa7414b0e5613e2fc37768a5/README.md#L250",
"source_id": "quant-redhatai-whisper-large-v3-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-quantized.w8a8/README.md",
"sha256": "9bb2fb57e15ed087049ba48528d75d7077c44dd6cd2ba541c5caeb083b409d60",
"line": 250,
"raw_line": " 2.0744 | ",
"row_label_line": 248,
"row_label": "test-clean",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-quantized.w8a8/README.md",
"sha256": "9bb2fb57e15ed087049ba48528d75d7077c44dd6cd2ba541c5caeb083b409d60",
"line": 249,
"raw_line": " 2.1725 | ",
"row_label_line": 248,
"row_label": "test-clean",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-quantized.w8a8@11d614f:librispeech",
"artifact": "RedHatAI/whisper-large-v3-quantized.w8a8",
"artifact_revision": "11d614fb6fc82faeaa7414b0e5613e2fc37768a5",
"direction": "lower-is-better",
"published_recovery_pct": 104.73
},
{
"model": "whisper-large-v3",
"model_id": "whisper-large-v3",
"params_b": 1.55,
"quant": "INT8 W8A8 (SmoothQuant + GPTQ via llm-compressor)",
"metric": "LibriSpeech test-other WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.03903,
"quant_score": 0.038955,
"retention": 1.0019252984212552,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-quantized.w8a8/blob/11d614fb6fc82faeaa7414b0e5613e2fc37768a5/README.md#L256",
"source_id": "quant-redhatai-whisper-large-v3-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-quantized.w8a8/README.md",
"sha256": "9bb2fb57e15ed087049ba48528d75d7077c44dd6cd2ba541c5caeb083b409d60",
"line": 256,
"raw_line": " 3.8955 | ",
"row_label_line": 254,
"row_label": "test-other",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-quantized.w8a8/README.md",
"sha256": "9bb2fb57e15ed087049ba48528d75d7077c44dd6cd2ba541c5caeb083b409d60",
"line": 255,
"raw_line": " 3.903 | ",
"row_label_line": 254,
"row_label": "test-other",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-quantized.w8a8@11d614f:librispeech",
"artifact": "RedHatAI/whisper-large-v3-quantized.w8a8",
"artifact_revision": "11d614fb6fc82faeaa7414b0e5613e2fc37768a5",
"direction": "lower-is-better",
"published_recovery_pct": 100.19
},
{
"model": "whisper-large-v3-turbo",
"model_id": "whisper-large-v3-turbo",
"params_b": 0.81,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC; table column header says \"w8a8\")",
"metric": "LibriSpeech test-clean WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.022668,
"quant_score": 0.022611,
"retention": 1.0025208969085844,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3-turbo, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-turbo-FP8-dynamic/blob/e72a6dca29d039a5c9ea13e622e496ca61e85c34/README.md#L182",
"source_id": "quant-redhatai-whisper-large-v3-turbo-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-FP8-dynamic/README.md",
"sha256": "187c5cf91e5cf6f398c9baa2266e46b766107a61dcb2221adeeee50e6e11b332",
"line": 182,
"raw_line": " 2.2611 | ",
"row_label_line": 180,
"row_label": "test-clean",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-FP8-dynamic/README.md",
"sha256": "187c5cf91e5cf6f398c9baa2266e46b766107a61dcb2221adeeee50e6e11b332",
"line": 181,
"raw_line": " 2.2668 | ",
"row_label_line": 180,
"row_label": "test-clean",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-turbo-FP8-dynamic@e72a6dc:librispeech",
"artifact": "RedHatAI/whisper-large-v3-turbo-FP8-dynamic",
"artifact_revision": "e72a6dca29d039a5c9ea13e622e496ca61e85c34",
"direction": "lower-is-better",
"published_recovery_pct": 100.25
},
{
"model": "whisper-large-v3-turbo",
"model_id": "whisper-large-v3-turbo",
"params_b": 0.81,
"quant": "FP8 dynamic (llm-compressor FP8_DYNAMIC; table column header says \"w8a8\")",
"metric": "LibriSpeech test-other WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.042396,
"quant_score": 0.043247,
"retention": 0.9803223344971906,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3-turbo, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-turbo-FP8-dynamic/blob/e72a6dca29d039a5c9ea13e622e496ca61e85c34/README.md#L188",
"source_id": "quant-redhatai-whisper-large-v3-turbo-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-FP8-dynamic/README.md",
"sha256": "187c5cf91e5cf6f398c9baa2266e46b766107a61dcb2221adeeee50e6e11b332",
"line": 188,
"raw_line": " 4.3247 | ",
"row_label_line": 186,
"row_label": "test-other",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-FP8-dynamic/README.md",
"sha256": "187c5cf91e5cf6f398c9baa2266e46b766107a61dcb2221adeeee50e6e11b332",
"line": 187,
"raw_line": " 4.2396 | ",
"row_label_line": 186,
"row_label": "test-other",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-turbo-FP8-dynamic@e72a6dc:librispeech",
"artifact": "RedHatAI/whisper-large-v3-turbo-FP8-dynamic",
"artifact_revision": "e72a6dca29d039a5c9ea13e622e496ca61e85c34",
"direction": "lower-is-better",
"published_recovery_pct": 98.03
},
{
"model": "whisper-large-v3-turbo",
"model_id": "whisper-large-v3-turbo",
"params_b": 0.81,
"quant": "INT4 W4A16 (GPTQ, weights only, group size 64 per the quantize.py command shown, actorder=weight)",
"metric": "LibriSpeech test-clean WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.021876,
"quant_score": 0.021951,
"retention": 0.996583299166325,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3-turbo, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-turbo-quantized.w4a16/blob/2009b1e9931565204da231c92ee9dce46034efc8/README.md#L426",
"source_id": "quant-redhatai-whisper-large-v3-turbo-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w4a16/README.md",
"sha256": "7ceff433ab21ff3f52a38987944b1d208211b1b19044d15c792dfc4037d4540b",
"line": 426,
"raw_line": " 2.1951 | ",
"row_label_line": 424,
"row_label": "test-clean",
"column_header": "W4A16",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w4a16/README.md",
"sha256": "7ceff433ab21ff3f52a38987944b1d208211b1b19044d15c792dfc4037d4540b",
"line": 425,
"raw_line": " 2.1876 | ",
"row_label_line": 424,
"row_label": "test-clean",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled. Note: this card re-measured its own baseline (all baseline cells differ from sibling cards: test-clean: 2.1876 vs 2.2668; test-other: 3.8992 vs 4.2396).",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-turbo-quantized.w4a16@2009b1e:librispeech",
"artifact": "RedHatAI/whisper-large-v3-turbo-quantized.w4a16",
"artifact_revision": "2009b1e9931565204da231c92ee9dce46034efc8",
"direction": "lower-is-better",
"published_recovery_pct": 99.66
},
{
"model": "whisper-large-v3-turbo",
"model_id": "whisper-large-v3-turbo",
"params_b": 0.81,
"quant": "INT4 W4A16 (GPTQ, weights only, group size 64 per the quantize.py command shown, actorder=weight)",
"metric": "LibriSpeech test-other WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.038992,
"quant_score": 0.040411,
"retention": 0.9648857984212219,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3-turbo, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-turbo-quantized.w4a16/blob/2009b1e9931565204da231c92ee9dce46034efc8/README.md#L432",
"source_id": "quant-redhatai-whisper-large-v3-turbo-quantized-w4a16",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w4a16/README.md",
"sha256": "7ceff433ab21ff3f52a38987944b1d208211b1b19044d15c792dfc4037d4540b",
"line": 432,
"raw_line": " 4.0411 | ",
"row_label_line": 430,
"row_label": "test-other",
"column_header": "W4A16",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w4a16/README.md",
"sha256": "7ceff433ab21ff3f52a38987944b1d208211b1b19044d15c792dfc4037d4540b",
"line": 431,
"raw_line": " 3.8992 | ",
"row_label_line": 430,
"row_label": "test-other",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled. Note: this card re-measured its own baseline (all baseline cells differ from sibling cards: test-clean: 2.1876 vs 2.2668; test-other: 3.8992 vs 4.2396).",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-turbo-quantized.w4a16@2009b1e:librispeech",
"artifact": "RedHatAI/whisper-large-v3-turbo-quantized.w4a16",
"artifact_revision": "2009b1e9931565204da231c92ee9dce46034efc8",
"direction": "lower-is-better",
"published_recovery_pct": 96.49
},
{
"model": "whisper-large-v3-turbo",
"model_id": "whisper-large-v3-turbo",
"params_b": 0.81,
"quant": "INT8 W8A8 (SmoothQuant + GPTQ via llm-compressor)",
"metric": "LibriSpeech test-clean WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.022668,
"quant_score": 0.02263,
"retention": 1.0016791869200177,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3-turbo, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-turbo-quantized.w8a8/blob/95991fd06170e03c8fa55595958972e6cb76c441/README.md#L250",
"source_id": "quant-redhatai-whisper-large-v3-turbo-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w8a8/README.md",
"sha256": "c284ba1b0944b232b334867a70a137315f221519a955e5b4624a56d48b46950a",
"line": 250,
"raw_line": " 2.263 | ",
"row_label_line": 248,
"row_label": "test-clean",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w8a8/README.md",
"sha256": "c284ba1b0944b232b334867a70a137315f221519a955e5b4624a56d48b46950a",
"line": 249,
"raw_line": " 2.2668 | ",
"row_label_line": 248,
"row_label": "test-clean",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-turbo-quantized.w8a8@95991fd:librispeech",
"artifact": "RedHatAI/whisper-large-v3-turbo-quantized.w8a8",
"artifact_revision": "95991fd06170e03c8fa55595958972e6cb76c441",
"direction": "lower-is-better",
"published_recovery_pct": 100.17
},
{
"model": "whisper-large-v3-turbo",
"model_id": "whisper-large-v3-turbo",
"params_b": 0.81,
"quant": "INT8 W8A8 (SmoothQuant + GPTQ via llm-compressor)",
"metric": "LibriSpeech test-other WER (lmms-eval `librispeech`, vLLM whisper backend, batch 64)",
"full": 0.042396,
"quant_score": 0.043134,
"retention": 0.9828905271943248,
"baseline_quant": "BF16 (unquantized openai/whisper-large-v3-turbo, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/whisper-large-v3-turbo-quantized.w8a8/blob/95991fd06170e03c8fa55595958972e6cb76c441/README.md#L256",
"source_id": "quant-redhatai-whisper-large-v3-turbo-quantized-w8a8",
"evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w8a8/README.md",
"sha256": "c284ba1b0944b232b334867a70a137315f221519a955e5b4624a56d48b46950a",
"line": 256,
"raw_line": " 4.3134 | ",
"row_label_line": 254,
"row_label": "test-other",
"column_header": "w8a8",
"category": "LibriSpeech (WER)"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__whisper-large-v3-turbo-quantized.w8a8/README.md",
"sha256": "c284ba1b0944b232b334867a70a137315f221519a955e5b4624a56d48b46950a",
"line": 255,
"raw_line": " 4.2396 | ",
"row_label_line": 254,
"row_label": "test-other",
"column_header": "BF16"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column.",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Fleurs rows are not recorded: the table labels them \"X→en\" which leaves the task (transcription vs translation) ambiguous. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/whisper-large-v3-turbo-quantized.w8a8@95991fd:librispeech",
"artifact": "RedHatAI/whisper-large-v3-turbo-quantized.w8a8",
"artifact_revision": "95991fd06170e03c8fa55595958972e6cb76c441",
"direction": "lower-is-better",
"published_recovery_pct": 98.29
},
{
"model": "Voxtral-Mini-3B-2507",
"model_id": "voxtral-mini-3b",
"params_b": 4.68,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "FLEURS English transcription WER",
"full": 0.0389,
"quant_score": 0.0395,
"retention": 0.9848101265822785,
"baseline_quant": "BF16 (unquantized mistralai/Voxtral-Mini-3B-2507, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Voxtral-Mini-3B-2507-FP8-dynamic/blob/b6694dd7dfc0d8c4bff962f04a39ffc98f50b2aa/README.md#L403",
"source_id": "quant-redhatai-voxtral-mini-3b-2507-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Voxtral-Mini-3B-2507-FP8-dynamic/README.md",
"sha256": "4d17439a3b1e1cf4195d45fddf7e1638ff3f963b950f4f4437c143ac5b6c024d",
"line": 403,
"raw_line": " 3.95% | ",
"row_label_line": 401,
"row_label": "English",
"column_header": "Voxtral-Mini-3B-2507-FP8-dynamic (this model)",
"category": "Fleurs WER"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Voxtral-Mini-3B-2507-FP8-dynamic/README.md",
"sha256": "4d17439a3b1e1cf4195d45fddf7e1638ff3f963b950f4f4437c143ac5b6c024d",
"line": 402,
"raw_line": " 3.89% | ",
"row_label_line": 401,
"row_label": "English",
"column_header": "Voxtral-Mini-3B-2507"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column (the card's Recovery uses (1-WER) ratios).",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Card computes Recovery on the complement of WER; our retention is baseline_wer/quant_wer instead. Other FLEURS languages in the table are not recorded as separate findings. Evaluation harness not named in the card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Voxtral-Mini-3B-2507-FP8-dynamic@b6694dd:fleurs",
"artifact": "RedHatAI/Voxtral-Mini-3B-2507-FP8-dynamic",
"artifact_revision": "b6694dd7dfc0d8c4bff962f04a39ffc98f50b2aa",
"direction": "lower-is-better",
"published_recovery_pct": 99.9
},
{
"model": "Voxtral-Small-24B-2507",
"model_id": "voxtral-small-24b",
"params_b": 24.3,
"quant": "FP8 W8A8 dynamic (weights symmetric static per-channel, activations symmetric dynamic per-token; llm-compressor FP8_DYNAMIC)",
"metric": "FLEURS English transcription WER",
"full": 0.0345,
"quant_score": 0.0343,
"retention": 1.0058309037900874,
"baseline_quant": "BF16 (unquantized mistralai/Voxtral-Small-24B-2507, evaluated in the same card study)",
"source": "https://huggingface.co/RedHatAI/Voxtral-Small-24B-2507-FP8-dynamic/blob/a5dba05c66c6f06ec358dbf6531087a858ab141d/README.md#L254",
"source_id": "quant-redhatai-voxtral-small-24b-2507-fp8-dynamic",
"evidence": {
"file": "speed-evidence/RedHatAI__Voxtral-Small-24B-2507-FP8-dynamic/README.md",
"sha256": "a5e1ade323c9e8b5239ee439201101bd18c9a9b2ffe84e2d9edc4bc058f7836f",
"line": 254,
"raw_line": " 3.43% | ",
"row_label_line": 252,
"row_label": "English",
"column_header": "Voxtral-Small-24B-2507-FP8-dynamic (this model)",
"category": "Fleurs WER"
},
"baseline_evidence": {
"file": "speed-evidence/RedHatAI__Voxtral-Small-24B-2507-FP8-dynamic/README.md",
"sha256": "a5e1ade323c9e8b5239ee439201101bd18c9a9b2ffe84e2d9edc4bc058f7836f",
"line": 253,
"raw_line": " 3.45% | ",
"row_label_line": 252,
"row_label": "English",
"column_header": "Voxtral-Small-24B-2507"
},
"transformation": "Published WER percentages / 100 (fractions of words). Lower is better: retention = baseline_wer / quant_wer (inverted so that <1 means the quantized model is worse); recomputed, not copied from the card's Recovery column (the card's Recovery uses (1-WER) ratios).",
"note": "Red Hat AI (Neural Magic) model-card evaluation of its own quantized checkpoint against the unquantized model in the same study. Card computes Recovery on the complement of WER; our retention is baseline_wer/quant_wer instead. Other FLEURS languages in the table are not recorded as separate findings. Evaluation harness not named in the card. Relative within-study evidence only; different harnesses/settings across cards are not pooled.",
"aggregation": "single benchmark",
"study": "RedHatAI/Voxtral-Small-24B-2507-FP8-dynamic@a5dba05:fleurs",
"artifact": "RedHatAI/Voxtral-Small-24B-2507-FP8-dynamic",
"artifact_revision": "a5dba05c66c6f06ec358dbf6531087a858ab141d",
"direction": "lower-is-better",
"published_recovery_pct": 100
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-0.6B-GPTQ-Int8)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.303,
"quant_score": 0.302,
"retention": 0.9966996699669967,
"baseline_quant": "bf16 (unquantized Qwen3-0.6B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-0.6B-GPTQ-Int8/blob/d3f20e7e71825fe57ad32705b396e9e156279107/README.md#L278",
"source_id": "quant-qwen-hf-qwen3-0-6b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 278,
"raw_line": "| Thinking | GPTQ-int8 | 30.2 | 29.3 | 53.8 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "30.2",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 277,
"raw_line": "| Thinking | bf16 | 30.3 | 27.9 | 55.6 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "30.3"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-0.6B-GPTQ-Int8 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-0.6B-GPTQ-Int8@d3f20e7:Performance",
"artifact": "Qwen/Qwen3-0.6B-GPTQ-Int8",
"artifact_revision": "d3f20e7e71825fe57ad32705b396e9e156279107"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-0.6B-GPTQ-Int8)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.279,
"quant_score": 0.293,
"retention": 1.050179211469534,
"baseline_quant": "bf16 (unquantized Qwen3-0.6B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-0.6B-GPTQ-Int8/blob/d3f20e7e71825fe57ad32705b396e9e156279107/README.md#L278",
"source_id": "quant-qwen-hf-qwen3-0-6b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 278,
"raw_line": "| Thinking | GPTQ-int8 | 30.2 | 29.3 | 53.8 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "29.3",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 277,
"raw_line": "| Thinking | bf16 | 30.3 | 27.9 | 55.6 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "27.9"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-0.6B-GPTQ-Int8 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-0.6B-GPTQ-Int8@d3f20e7:Performance",
"artifact": "Qwen/Qwen3-0.6B-GPTQ-Int8",
"artifact_revision": "d3f20e7e71825fe57ad32705b396e9e156279107"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-0.6B-GPTQ-Int8)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.556,
"quant_score": 0.538,
"retention": 0.9676258992805753,
"baseline_quant": "bf16 (unquantized Qwen3-0.6B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-0.6B-GPTQ-Int8/blob/d3f20e7e71825fe57ad32705b396e9e156279107/README.md#L278",
"source_id": "quant-qwen-hf-qwen3-0-6b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 278,
"raw_line": "| Thinking | GPTQ-int8 | 30.2 | 29.3 | 53.8 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "53.8",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 277,
"raw_line": "| Thinking | bf16 | 30.3 | 27.9 | 55.6 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "55.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-0.6B-GPTQ-Int8 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-0.6B-GPTQ-Int8@d3f20e7:Performance",
"artifact": "Qwen/Qwen3-0.6B-GPTQ-Int8",
"artifact_revision": "d3f20e7e71825fe57ad32705b396e9e156279107"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-0.6B-GPTQ-Int8)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.218,
"quant_score": 0.218,
"retention": 1,
"baseline_quant": "bf16 (unquantized Qwen3-0.6B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-0.6B-GPTQ-Int8/blob/d3f20e7e71825fe57ad32705b396e9e156279107/README.md#L280",
"source_id": "quant-qwen-hf-qwen3-0-6b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 280,
"raw_line": "| Non-Thinking | GPTQ-int8 | 21.8 | 23.2 | 44.9 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "21.8",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 279,
"raw_line": "| Non-Thinking | bf16 | 21.8 | 22.9 | 44.6 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "21.8"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-0.6B-GPTQ-Int8 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-0.6B-GPTQ-Int8@d3f20e7:Performance",
"artifact": "Qwen/Qwen3-0.6B-GPTQ-Int8",
"artifact_revision": "d3f20e7e71825fe57ad32705b396e9e156279107"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-0.6B-GPTQ-Int8)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.229,
"quant_score": 0.232,
"retention": 1.0131004366812226,
"baseline_quant": "bf16 (unquantized Qwen3-0.6B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-0.6B-GPTQ-Int8/blob/d3f20e7e71825fe57ad32705b396e9e156279107/README.md#L280",
"source_id": "quant-qwen-hf-qwen3-0-6b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 280,
"raw_line": "| Non-Thinking | GPTQ-int8 | 21.8 | 23.2 | 44.9 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "23.2",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 279,
"raw_line": "| Non-Thinking | bf16 | 21.8 | 22.9 | 44.6 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "22.9"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-0.6B-GPTQ-Int8 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-0.6B-GPTQ-Int8@d3f20e7:Performance",
"artifact": "Qwen/Qwen3-0.6B-GPTQ-Int8",
"artifact_revision": "d3f20e7e71825fe57ad32705b396e9e156279107"
},
{
"model": "Qwen3-0.6B",
"model_id": "qwen3-0-6b",
"params_b": 0.75,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-0.6B-GPTQ-Int8)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.446,
"quant_score": 0.449,
"retention": 1.006726457399103,
"baseline_quant": "bf16 (unquantized Qwen3-0.6B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-0.6B-GPTQ-Int8/blob/d3f20e7e71825fe57ad32705b396e9e156279107/README.md#L280",
"source_id": "quant-qwen-hf-qwen3-0-6b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 280,
"raw_line": "| Non-Thinking | GPTQ-int8 | 21.8 | 23.2 | 44.9 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "44.9",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-0.6B-GPTQ-Int8/README.md",
"sha256": "c57978904602fec80b56a50a4123ff44225219624ea5b7d6943f0b87fc13cb51",
"line": 279,
"raw_line": "| Non-Thinking | bf16 | 21.8 | 22.9 | 44.6 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "44.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-0.6B-GPTQ-Int8 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-0.6B-GPTQ-Int8@d3f20e7:Performance",
"artifact": "Qwen/Qwen3-0.6B-GPTQ-Int8",
"artifact_revision": "d3f20e7e71825fe57ad32705b396e9e156279107"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-1.7B-GPTQ-Int8)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.511,
"quant_score": 0.498,
"retention": 0.974559686888454,
"baseline_quant": "bf16 (unquantized Qwen3-1.7B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-1.7B-GPTQ-Int8/blob/382fed3fa21d1a7e837b42a500564a0c4be4f060/README.md#L278",
"source_id": "quant-qwen-hf-qwen3-1-7b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 278,
"raw_line": "| Thinking | GPTQ-int8 | 49.8 | 39.1 | 74.9 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "49.8",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 277,
"raw_line": "| Thinking | bf16 | 51.1 | 40.1 | 73.9 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "51.1"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-1.7B-GPTQ-Int8 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-1.7B-GPTQ-Int8@382fed3:Performance",
"artifact": "Qwen/Qwen3-1.7B-GPTQ-Int8",
"artifact_revision": "382fed3fa21d1a7e837b42a500564a0c4be4f060"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-1.7B-GPTQ-Int8)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.401,
"quant_score": 0.391,
"retention": 0.9750623441396509,
"baseline_quant": "bf16 (unquantized Qwen3-1.7B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-1.7B-GPTQ-Int8/blob/382fed3fa21d1a7e837b42a500564a0c4be4f060/README.md#L278",
"source_id": "quant-qwen-hf-qwen3-1-7b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 278,
"raw_line": "| Thinking | GPTQ-int8 | 49.8 | 39.1 | 74.9 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "39.1",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 277,
"raw_line": "| Thinking | bf16 | 51.1 | 40.1 | 73.9 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "40.1"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-1.7B-GPTQ-Int8 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-1.7B-GPTQ-Int8@382fed3:Performance",
"artifact": "Qwen/Qwen3-1.7B-GPTQ-Int8",
"artifact_revision": "382fed3fa21d1a7e837b42a500564a0c4be4f060"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-1.7B-GPTQ-Int8)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.739,
"quant_score": 0.749,
"retention": 1.013531799729364,
"baseline_quant": "bf16 (unquantized Qwen3-1.7B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-1.7B-GPTQ-Int8/blob/382fed3fa21d1a7e837b42a500564a0c4be4f060/README.md#L278",
"source_id": "quant-qwen-hf-qwen3-1-7b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 278,
"raw_line": "| Thinking | GPTQ-int8 | 49.8 | 39.1 | 74.9 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "74.9",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 277,
"raw_line": "| Thinking | bf16 | 51.1 | 40.1 | 73.9 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "73.9"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-1.7B-GPTQ-Int8 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-1.7B-GPTQ-Int8@382fed3:Performance",
"artifact": "Qwen/Qwen3-1.7B-GPTQ-Int8",
"artifact_revision": "382fed3fa21d1a7e837b42a500564a0c4be4f060"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-1.7B-GPTQ-Int8)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.356,
"quant_score": 0.355,
"retention": 0.9971910112359549,
"baseline_quant": "bf16 (unquantized Qwen3-1.7B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-1.7B-GPTQ-Int8/blob/382fed3fa21d1a7e837b42a500564a0c4be4f060/README.md#L280",
"source_id": "quant-qwen-hf-qwen3-1-7b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 280,
"raw_line": "| Non-Thinking | GPTQ-int8 | 35.5 | 28.3 | 63.0 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "35.5",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 279,
"raw_line": "| Non-Thinking | bf16 | 35.6 | 28.6 | 64.4 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "35.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-1.7B-GPTQ-Int8 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-1.7B-GPTQ-Int8@382fed3:Performance",
"artifact": "Qwen/Qwen3-1.7B-GPTQ-Int8",
"artifact_revision": "382fed3fa21d1a7e837b42a500564a0c4be4f060"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-1.7B-GPTQ-Int8)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.286,
"quant_score": 0.283,
"retention": 0.9895104895104895,
"baseline_quant": "bf16 (unquantized Qwen3-1.7B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-1.7B-GPTQ-Int8/blob/382fed3fa21d1a7e837b42a500564a0c4be4f060/README.md#L280",
"source_id": "quant-qwen-hf-qwen3-1-7b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 280,
"raw_line": "| Non-Thinking | GPTQ-int8 | 35.5 | 28.3 | 63.0 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "28.3",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 279,
"raw_line": "| Non-Thinking | bf16 | 35.6 | 28.6 | 64.4 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "28.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-1.7B-GPTQ-Int8 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-1.7B-GPTQ-Int8@382fed3:Performance",
"artifact": "Qwen/Qwen3-1.7B-GPTQ-Int8",
"artifact_revision": "382fed3fa21d1a7e837b42a500564a0c4be4f060"
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GPTQ INT8 (w8, group_size 128, sym, desc_act=false; GPTQModel 2.2.0; checkpoint Qwen/Qwen3-1.7B-GPTQ-Int8)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.644,
"quant_score": 0.63,
"retention": 0.9782608695652174,
"baseline_quant": "bf16 (unquantized Qwen3-1.7B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-1.7B-GPTQ-Int8/blob/382fed3fa21d1a7e837b42a500564a0c4be4f060/README.md#L280",
"source_id": "quant-qwen-hf-qwen3-1-7b-gptq-int8",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 280,
"raw_line": "| Non-Thinking | GPTQ-int8 | 35.5 | 28.3 | 63.0 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "63.0",
"header_line": 275
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-1.7B-GPTQ-Int8/README.md",
"sha256": "3cac7b6c94f7e8f1ba0e4cc0473afc0bc104c0fea4c58d47465f4cb0a7bce0bc",
"line": 279,
"raw_line": "| Non-Thinking | bf16 | 35.6 | 28.6 | 64.4 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "64.4"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-1.7B-GPTQ-Int8 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-1.7B-GPTQ-Int8@382fed3:Performance",
"artifact": "Qwen/Qwen3-1.7B-GPTQ-Int8",
"artifact_revision": "382fed3fa21d1a7e837b42a500564a0c4be4f060"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-4B-AWQ)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.636,
"quant_score": 0.617,
"retention": 0.970125786163522,
"baseline_quant": "bf16 (unquantized Qwen3-4B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-4B-AWQ/blob/74d4bd2bd4bff9cafc9345221320bffb08b406a3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-4b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 61.7 | 54.4 | 82.0 | 65.0 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "61.7",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 323,
"raw_line": "| Thinking | bf16 | 63.6 | 55.9 | 83.7 | 73.8 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "63.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-4B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-4B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-4B-AWQ@74d4bd2:Performance",
"artifact": "Qwen/Qwen3-4B-AWQ",
"artifact_revision": "74d4bd2bd4bff9cafc9345221320bffb08b406a3"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-4B-AWQ)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.559,
"quant_score": 0.544,
"retention": 0.9731663685152059,
"baseline_quant": "bf16 (unquantized Qwen3-4B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-4B-AWQ/blob/74d4bd2bd4bff9cafc9345221320bffb08b406a3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-4b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 61.7 | 54.4 | 82.0 | 65.0 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "54.4",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 323,
"raw_line": "| Thinking | bf16 | 63.6 | 55.9 | 83.7 | 73.8 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "55.9"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-4B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-4B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-4B-AWQ@74d4bd2:Performance",
"artifact": "Qwen/Qwen3-4B-AWQ",
"artifact_revision": "74d4bd2bd4bff9cafc9345221320bffb08b406a3"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-4B-AWQ)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.837,
"quant_score": 0.82,
"retention": 0.9796893667861408,
"baseline_quant": "bf16 (unquantized Qwen3-4B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-4B-AWQ/blob/74d4bd2bd4bff9cafc9345221320bffb08b406a3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-4b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 61.7 | 54.4 | 82.0 | 65.0 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "82.0",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 323,
"raw_line": "| Thinking | bf16 | 63.6 | 55.9 | 83.7 | 73.8 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "83.7"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-4B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-4B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-4B-AWQ@74d4bd2:Performance",
"artifact": "Qwen/Qwen3-4B-AWQ",
"artifact_revision": "74d4bd2bd4bff9cafc9345221320bffb08b406a3"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-4B-AWQ)",
"metric": "AIME 2024, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.738,
"quant_score": 0.65,
"retention": 0.8807588075880759,
"baseline_quant": "bf16 (unquantized Qwen3-4B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-4B-AWQ/blob/74d4bd2bd4bff9cafc9345221320bffb08b406a3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-4b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 61.7 | 54.4 | 82.0 | 65.0 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "65.0",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 323,
"raw_line": "| Thinking | bf16 | 63.6 | 55.9 | 83.7 | 73.8 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "73.8"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-4B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-4B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-4B-AWQ@74d4bd2:Performance",
"artifact": "Qwen/Qwen3-4B-AWQ",
"artifact_revision": "74d4bd2bd4bff9cafc9345221320bffb08b406a3"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-4B-AWQ)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.484,
"quant_score": 0.461,
"retention": 0.9524793388429753,
"baseline_quant": "bf16 (unquantized Qwen3-4B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-4B-AWQ/blob/74d4bd2bd4bff9cafc9345221320bffb08b406a3/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-4b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 46.1 | 40.4 | 73.4 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "46.1",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 48.4 | 41.7 | 77.3 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "48.4"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-4B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-4B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-4B-AWQ@74d4bd2:Performance",
"artifact": "Qwen/Qwen3-4B-AWQ",
"artifact_revision": "74d4bd2bd4bff9cafc9345221320bffb08b406a3"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-4B-AWQ)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.417,
"quant_score": 0.404,
"retention": 0.9688249400479615,
"baseline_quant": "bf16 (unquantized Qwen3-4B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-4B-AWQ/blob/74d4bd2bd4bff9cafc9345221320bffb08b406a3/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-4b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 46.1 | 40.4 | 73.4 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "40.4",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 48.4 | 41.7 | 77.3 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "41.7"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-4B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-4B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-4B-AWQ@74d4bd2:Performance",
"artifact": "Qwen/Qwen3-4B-AWQ",
"artifact_revision": "74d4bd2bd4bff9cafc9345221320bffb08b406a3"
},
{
"model": "Qwen3-4B",
"model_id": null,
"params_b": 4,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-4B-AWQ)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.773,
"quant_score": 0.734,
"retention": 0.9495472186287194,
"baseline_quant": "bf16 (unquantized Qwen3-4B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-4B-AWQ/blob/74d4bd2bd4bff9cafc9345221320bffb08b406a3/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-4b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 46.1 | 40.4 | 73.4 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "73.4",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-4B-AWQ/README.md",
"sha256": "7e2971f6a961c63b43b810cceed23d4e318b3762079fa08f0bb4192cd80b883f",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 48.4 | 41.7 | 77.3 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "77.3"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-4B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-4B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-4B-AWQ@74d4bd2:Performance",
"artifact": "Qwen/Qwen3-4B-AWQ",
"artifact_revision": "74d4bd2bd4bff9cafc9345221320bffb08b406a3"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; AutoAWQ; checkpoint Qwen/Qwen3-8B-AWQ)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.671,
"quant_score": 0.655,
"retention": 0.9761549925484353,
"baseline_quant": "bf16 (unquantized Qwen3-8B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-8B-AWQ/blob/4da05a8edb55c6046cce958586c33b61da07bb79/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-8b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 65.5 | 59.0 | 86.4 | 71.3 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "65.5",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 323,
"raw_line": "| Thinking | bf16 | 67.1 | 62.0 | 87.5 | 76.0 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "67.1"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-8B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-8B-AWQ@4da05a8:Performance",
"artifact": "Qwen/Qwen3-8B-AWQ",
"artifact_revision": "4da05a8edb55c6046cce958586c33b61da07bb79"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; AutoAWQ; checkpoint Qwen/Qwen3-8B-AWQ)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.62,
"quant_score": 0.59,
"retention": 0.9516129032258064,
"baseline_quant": "bf16 (unquantized Qwen3-8B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-8B-AWQ/blob/4da05a8edb55c6046cce958586c33b61da07bb79/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-8b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 65.5 | 59.0 | 86.4 | 71.3 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "59.0",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 323,
"raw_line": "| Thinking | bf16 | 67.1 | 62.0 | 87.5 | 76.0 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "62.0"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-8B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-8B-AWQ@4da05a8:Performance",
"artifact": "Qwen/Qwen3-8B-AWQ",
"artifact_revision": "4da05a8edb55c6046cce958586c33b61da07bb79"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; AutoAWQ; checkpoint Qwen/Qwen3-8B-AWQ)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.875,
"quant_score": 0.864,
"retention": 0.9874285714285715,
"baseline_quant": "bf16 (unquantized Qwen3-8B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-8B-AWQ/blob/4da05a8edb55c6046cce958586c33b61da07bb79/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-8b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 65.5 | 59.0 | 86.4 | 71.3 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "86.4",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 323,
"raw_line": "| Thinking | bf16 | 67.1 | 62.0 | 87.5 | 76.0 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "87.5"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-8B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-8B-AWQ@4da05a8:Performance",
"artifact": "Qwen/Qwen3-8B-AWQ",
"artifact_revision": "4da05a8edb55c6046cce958586c33b61da07bb79"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; AutoAWQ; checkpoint Qwen/Qwen3-8B-AWQ)",
"metric": "AIME 2024, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.76,
"quant_score": 0.713,
"retention": 0.9381578947368421,
"baseline_quant": "bf16 (unquantized Qwen3-8B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-8B-AWQ/blob/4da05a8edb55c6046cce958586c33b61da07bb79/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-8b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 65.5 | 59.0 | 86.4 | 71.3 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "71.3",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 323,
"raw_line": "| Thinking | bf16 | 67.1 | 62.0 | 87.5 | 76.0 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "76.0"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-8B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-8B-AWQ@4da05a8:Performance",
"artifact": "Qwen/Qwen3-8B-AWQ",
"artifact_revision": "4da05a8edb55c6046cce958586c33b61da07bb79"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; AutoAWQ; checkpoint Qwen/Qwen3-8B-AWQ)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.535,
"quant_score": 0.489,
"retention": 0.914018691588785,
"baseline_quant": "bf16 (unquantized Qwen3-8B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-8B-AWQ/blob/4da05a8edb55c6046cce958586c33b61da07bb79/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-8b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 48.9 | 35.9 | 79.1 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "48.9",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 53.5 | 39.3 | 79.5 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "53.5"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-8B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-8B-AWQ@4da05a8:Performance",
"artifact": "Qwen/Qwen3-8B-AWQ",
"artifact_revision": "4da05a8edb55c6046cce958586c33b61da07bb79"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; AutoAWQ; checkpoint Qwen/Qwen3-8B-AWQ)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.393,
"quant_score": 0.359,
"retention": 0.9134860050890585,
"baseline_quant": "bf16 (unquantized Qwen3-8B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-8B-AWQ/blob/4da05a8edb55c6046cce958586c33b61da07bb79/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-8b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 48.9 | 35.9 | 79.1 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "35.9",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 53.5 | 39.3 | 79.5 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "39.3"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-8B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-8B-AWQ@4da05a8:Performance",
"artifact": "Qwen/Qwen3-8B-AWQ",
"artifact_revision": "4da05a8edb55c6046cce958586c33b61da07bb79"
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; AutoAWQ; checkpoint Qwen/Qwen3-8B-AWQ)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.795,
"quant_score": 0.791,
"retention": 0.9949685534591194,
"baseline_quant": "bf16 (unquantized Qwen3-8B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-8B-AWQ/blob/4da05a8edb55c6046cce958586c33b61da07bb79/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-8b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 48.9 | 35.9 | 79.1 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "79.1",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-8B-AWQ/README.md",
"sha256": "5c397e99916138e00686567882ea1308524c95c2962f05d0ef57ebfa0d2c81fb",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 53.5 | 39.3 | 79.5 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "79.5"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-8B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-8B-AWQ@4da05a8:Performance",
"artifact": "Qwen/Qwen3-8B-AWQ",
"artifact_revision": "4da05a8edb55c6046cce958586c33b61da07bb79"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-14B-AWQ)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.713,
"quant_score": 0.7,
"retention": 0.9817671809256662,
"baseline_quant": "bf16 (unquantized Qwen3-14B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-14B-AWQ/blob/31c69efc29464b6bb0aee1398b5a7b50a99340c3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-14b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 70.0 | 62.1 | 88.5 | 77.0 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "70.0",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 323,
"raw_line": "| Thinking | bf16 | 71.3 | 64.0 | 88.6 | 79.3 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "71.3"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-14B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-14B-AWQ@31c69ef:Performance",
"artifact": "Qwen/Qwen3-14B-AWQ",
"artifact_revision": "31c69efc29464b6bb0aee1398b5a7b50a99340c3"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-14B-AWQ)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.64,
"quant_score": 0.621,
"retention": 0.9703125,
"baseline_quant": "bf16 (unquantized Qwen3-14B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-14B-AWQ/blob/31c69efc29464b6bb0aee1398b5a7b50a99340c3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-14b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 70.0 | 62.1 | 88.5 | 77.0 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "62.1",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 323,
"raw_line": "| Thinking | bf16 | 71.3 | 64.0 | 88.6 | 79.3 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "64.0"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-14B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-14B-AWQ@31c69ef:Performance",
"artifact": "Qwen/Qwen3-14B-AWQ",
"artifact_revision": "31c69efc29464b6bb0aee1398b5a7b50a99340c3"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-14B-AWQ)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.886,
"quant_score": 0.885,
"retention": 0.9988713318284426,
"baseline_quant": "bf16 (unquantized Qwen3-14B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-14B-AWQ/blob/31c69efc29464b6bb0aee1398b5a7b50a99340c3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-14b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 70.0 | 62.1 | 88.5 | 77.0 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "88.5",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 323,
"raw_line": "| Thinking | bf16 | 71.3 | 64.0 | 88.6 | 79.3 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "88.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-14B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-14B-AWQ@31c69ef:Performance",
"artifact": "Qwen/Qwen3-14B-AWQ",
"artifact_revision": "31c69efc29464b6bb0aee1398b5a7b50a99340c3"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-14B-AWQ)",
"metric": "AIME 2024, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.793,
"quant_score": 0.77,
"retention": 0.9709962168978563,
"baseline_quant": "bf16 (unquantized Qwen3-14B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-14B-AWQ/blob/31c69efc29464b6bb0aee1398b5a7b50a99340c3/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-14b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 70.0 | 62.1 | 88.5 | 77.0 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "77.0",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 323,
"raw_line": "| Thinking | bf16 | 71.3 | 64.0 | 88.6 | 79.3 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "79.3"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-14B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-14B-AWQ@31c69ef:Performance",
"artifact": "Qwen/Qwen3-14B-AWQ",
"artifact_revision": "31c69efc29464b6bb0aee1398b5a7b50a99340c3"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-14B-AWQ)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.596,
"quant_score": 0.574,
"retention": 0.9630872483221476,
"baseline_quant": "bf16 (unquantized Qwen3-14B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-14B-AWQ/blob/31c69efc29464b6bb0aee1398b5a7b50a99340c3/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-14b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 57.4 | 53.8 | 81.5 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "57.4",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 59.6 | 54.8 | 82.0 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "59.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-14B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-14B-AWQ@31c69ef:Performance",
"artifact": "Qwen/Qwen3-14B-AWQ",
"artifact_revision": "31c69efc29464b6bb0aee1398b5a7b50a99340c3"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-14B-AWQ)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.548,
"quant_score": 0.538,
"retention": 0.9817518248175182,
"baseline_quant": "bf16 (unquantized Qwen3-14B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-14B-AWQ/blob/31c69efc29464b6bb0aee1398b5a7b50a99340c3/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-14b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 57.4 | 53.8 | 81.5 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "53.8",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 59.6 | 54.8 | 82.0 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "54.8"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-14B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-14B-AWQ@31c69ef:Performance",
"artifact": "Qwen/Qwen3-14B-AWQ",
"artifact_revision": "31c69efc29464b6bb0aee1398b5a7b50a99340c3"
},
{
"model": "Qwen3-14B",
"model_id": "qwen3-14b",
"params_b": 14.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-14B-AWQ)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.82,
"quant_score": 0.815,
"retention": 0.9939024390243902,
"baseline_quant": "bf16 (unquantized Qwen3-14B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-14B-AWQ/blob/31c69efc29464b6bb0aee1398b5a7b50a99340c3/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-14b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 57.4 | 53.8 | 81.5 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "81.5",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-14B-AWQ/README.md",
"sha256": "0d3e27f85167f83769abfb5eb6ad2069be4a1bc7835cfec1612dd1aa1ac546fa",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 59.6 | 54.8 | 82.0 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "82.0"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-14B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-14B-AWQ@31c69ef:Performance",
"artifact": "Qwen/Qwen3-14B-AWQ",
"artifact_revision": "31c69efc29464b6bb0aee1398b5a7b50a99340c3"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-32B-AWQ)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.749,
"quant_score": 0.731,
"retention": 0.9759679572763683,
"baseline_quant": "bf16 (unquantized Qwen3-32B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-32B-AWQ/blob/0499c3ac83fdef8810b907a23894ba91e95eddd8/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-32b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 73.1 | 69.0 | 90.8 | 79.4 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "73.1",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 323,
"raw_line": "| Thinking | bf16 | 74.9 | 68.4 | 90.9 | 81.4 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "74.9"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-32B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-32B-AWQ@0499c3a:Performance",
"artifact": "Qwen/Qwen3-32B-AWQ",
"artifact_revision": "0499c3ac83fdef8810b907a23894ba91e95eddd8"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-32B-AWQ)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.684,
"quant_score": 0.69,
"retention": 1.0087719298245612,
"baseline_quant": "bf16 (unquantized Qwen3-32B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-32B-AWQ/blob/0499c3ac83fdef8810b907a23894ba91e95eddd8/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-32b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 73.1 | 69.0 | 90.8 | 79.4 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "69.0",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 323,
"raw_line": "| Thinking | bf16 | 74.9 | 68.4 | 90.9 | 81.4 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "68.4"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-32B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-32B-AWQ@0499c3a:Performance",
"artifact": "Qwen/Qwen3-32B-AWQ",
"artifact_revision": "0499c3ac83fdef8810b907a23894ba91e95eddd8"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-32B-AWQ)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.909,
"quant_score": 0.908,
"retention": 0.9988998899889988,
"baseline_quant": "bf16 (unquantized Qwen3-32B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-32B-AWQ/blob/0499c3ac83fdef8810b907a23894ba91e95eddd8/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-32b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 73.1 | 69.0 | 90.8 | 79.4 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "90.8",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 323,
"raw_line": "| Thinking | bf16 | 74.9 | 68.4 | 90.9 | 81.4 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "90.9"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-32B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-32B-AWQ@0499c3a:Performance",
"artifact": "Qwen/Qwen3-32B-AWQ",
"artifact_revision": "0499c3ac83fdef8810b907a23894ba91e95eddd8"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-32B-AWQ)",
"metric": "AIME 2024, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.814,
"quant_score": 0.794,
"retention": 0.9754299754299754,
"baseline_quant": "bf16 (unquantized Qwen3-32B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-32B-AWQ/blob/0499c3ac83fdef8810b907a23894ba91e95eddd8/README.md#L324",
"source_id": "quant-qwen-hf-qwen3-32b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 324,
"raw_line": "| Thinking | AWQ-int4 | 73.1 | 69.0 | 90.8 | 79.4 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "79.4",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 323,
"raw_line": "| Thinking | bf16 | 74.9 | 68.4 | 90.9 | 81.4 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "81.4"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-32B-AWQ card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-32B-AWQ@0499c3a:Performance",
"artifact": "Qwen/Qwen3-32B-AWQ",
"artifact_revision": "0499c3ac83fdef8810b907a23894ba91e95eddd8"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-32B-AWQ)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.598,
"quant_score": 0.598,
"retention": 1,
"baseline_quant": "bf16 (unquantized Qwen3-32B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-32B-AWQ/blob/0499c3ac83fdef8810b907a23894ba91e95eddd8/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-32b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 59.8 | 53.1 | 85.6 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "59.8",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 59.8 | 54.6 | 85.7 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "59.8"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-32B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-32B-AWQ@0499c3a:Performance",
"artifact": "Qwen/Qwen3-32B-AWQ",
"artifact_revision": "0499c3ac83fdef8810b907a23894ba91e95eddd8"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-32B-AWQ)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.546,
"quant_score": 0.531,
"retention": 0.9725274725274725,
"baseline_quant": "bf16 (unquantized Qwen3-32B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-32B-AWQ/blob/0499c3ac83fdef8810b907a23894ba91e95eddd8/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-32b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 59.8 | 53.1 | 85.6 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "53.1",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 59.8 | 54.6 | 85.7 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "54.6"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-32B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-32B-AWQ@0499c3a:Performance",
"artifact": "Qwen/Qwen3-32B-AWQ",
"artifact_revision": "0499c3ac83fdef8810b907a23894ba91e95eddd8"
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ INT4 (w4, group_size 128, zero_point, GEMM; checkpoint Qwen/Qwen3-32B-AWQ)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.857,
"quant_score": 0.856,
"retention": 0.9988331388564761,
"baseline_quant": "bf16 (unquantized Qwen3-32B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-32B-AWQ/blob/0499c3ac83fdef8810b907a23894ba91e95eddd8/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-32b-awq",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 326,
"raw_line": "| Non-Thinking | AWQ-int4 | 59.8 | 53.1 | 85.6 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "85.6",
"header_line": 321
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-32B-AWQ/README.md",
"sha256": "45c09c19deb406f822b42f3dbefd8905bcdd4da6706c925a3858e5aacb6ff827",
"line": 325,
"raw_line": "| Non-Thinking | bf16 | 59.8 | 54.6 | 85.7 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "85.7"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-32B-AWQ card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-32B-AWQ@0499c3a:Performance",
"artifact": "Qwen/Qwen3-32B-AWQ",
"artifact_revision": "0499c3ac83fdef8810b907a23894ba91e95eddd8"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-30B-A3B-GPTQ-Int4)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.743,
"quant_score": 0.715,
"retention": 0.962314939434724,
"baseline_quant": "bf16 (unquantized Qwen3-30B-A3B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4/blob/9b534e4318b7ebc3c961a839f13eb18b1833f441/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-30b-a3b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 326,
"raw_line": "| Thinking | GPTQ-int4 | 71.5 | 60.1 | 88.8 | 79.5 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "71.5",
"header_line": 323
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 325,
"raw_line": "| Thinking | bf16 | 74.3 | 65.8 | 89.5 | 80.4 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "74.3"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-30B-A3B-GPTQ-Int4 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-30B-A3B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-30B-A3B-GPTQ-Int4@9b534e4:Performance",
"artifact": "Qwen/Qwen3-30B-A3B-GPTQ-Int4",
"artifact_revision": "9b534e4318b7ebc3c961a839f13eb18b1833f441"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-30B-A3B-GPTQ-Int4)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.658,
"quant_score": 0.601,
"retention": 0.9133738601823709,
"baseline_quant": "bf16 (unquantized Qwen3-30B-A3B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4/blob/9b534e4318b7ebc3c961a839f13eb18b1833f441/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-30b-a3b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 326,
"raw_line": "| Thinking | GPTQ-int4 | 71.5 | 60.1 | 88.8 | 79.5 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "60.1",
"header_line": 323
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 325,
"raw_line": "| Thinking | bf16 | 74.3 | 65.8 | 89.5 | 80.4 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "65.8"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-30B-A3B-GPTQ-Int4 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-30B-A3B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-30B-A3B-GPTQ-Int4@9b534e4:Performance",
"artifact": "Qwen/Qwen3-30B-A3B-GPTQ-Int4",
"artifact_revision": "9b534e4318b7ebc3c961a839f13eb18b1833f441"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-30B-A3B-GPTQ-Int4)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.895,
"quant_score": 0.888,
"retention": 0.9921787709497206,
"baseline_quant": "bf16 (unquantized Qwen3-30B-A3B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4/blob/9b534e4318b7ebc3c961a839f13eb18b1833f441/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-30b-a3b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 326,
"raw_line": "| Thinking | GPTQ-int4 | 71.5 | 60.1 | 88.8 | 79.5 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "88.8",
"header_line": 323
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 325,
"raw_line": "| Thinking | bf16 | 74.3 | 65.8 | 89.5 | 80.4 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "89.5"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-30B-A3B-GPTQ-Int4 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-30B-A3B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-30B-A3B-GPTQ-Int4@9b534e4:Performance",
"artifact": "Qwen/Qwen3-30B-A3B-GPTQ-Int4",
"artifact_revision": "9b534e4318b7ebc3c961a839f13eb18b1833f441"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-30B-A3B-GPTQ-Int4)",
"metric": "AIME 2024, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.804,
"quant_score": 0.795,
"retention": 0.9888059701492538,
"baseline_quant": "bf16 (unquantized Qwen3-30B-A3B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4/blob/9b534e4318b7ebc3c961a839f13eb18b1833f441/README.md#L326",
"source_id": "quant-qwen-hf-qwen3-30b-a3b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 326,
"raw_line": "| Thinking | GPTQ-int4 | 71.5 | 60.1 | 88.8 | 79.5 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "79.5",
"header_line": 323
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 325,
"raw_line": "| Thinking | bf16 | 74.3 | 65.8 | 89.5 | 80.4 |",
"column_header": "AIME24",
"column": 5,
"source_cell": "80.4"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-30B-A3B-GPTQ-Int4 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-30B-A3B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-30B-A3B-GPTQ-Int4@9b534e4:Performance",
"artifact": "Qwen/Qwen3-30B-A3B-GPTQ-Int4",
"artifact_revision": "9b534e4318b7ebc3c961a839f13eb18b1833f441"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-30B-A3B-GPTQ-Int4)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.594,
"quant_score": 0.572,
"retention": 0.9629629629629631,
"baseline_quant": "bf16 (unquantized Qwen3-30B-A3B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4/blob/9b534e4318b7ebc3c961a839f13eb18b1833f441/README.md#L328",
"source_id": "quant-qwen-hf-qwen3-30b-a3b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 328,
"raw_line": "| Non-Thinking | GPTQ-int4 | 57.2 | 50.4 | 83.5 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "57.2",
"header_line": 323
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 327,
"raw_line": "| Non-Thinking | bf16 | 59.4 | 54.8 | 84.1 | - |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "59.4"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-30B-A3B-GPTQ-Int4 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-30B-A3B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-30B-A3B-GPTQ-Int4@9b534e4:Performance",
"artifact": "Qwen/Qwen3-30B-A3B-GPTQ-Int4",
"artifact_revision": "9b534e4318b7ebc3c961a839f13eb18b1833f441"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-30B-A3B-GPTQ-Int4)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.548,
"quant_score": 0.504,
"retention": 0.9197080291970804,
"baseline_quant": "bf16 (unquantized Qwen3-30B-A3B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4/blob/9b534e4318b7ebc3c961a839f13eb18b1833f441/README.md#L328",
"source_id": "quant-qwen-hf-qwen3-30b-a3b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 328,
"raw_line": "| Non-Thinking | GPTQ-int4 | 57.2 | 50.4 | 83.5 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "50.4",
"header_line": 323
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 327,
"raw_line": "| Non-Thinking | bf16 | 59.4 | 54.8 | 84.1 | - |",
"column_header": "GPQA",
"column": 3,
"source_cell": "54.8"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-30B-A3B-GPTQ-Int4 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-30B-A3B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-30B-A3B-GPTQ-Int4@9b534e4:Performance",
"artifact": "Qwen/Qwen3-30B-A3B-GPTQ-Int4",
"artifact_revision": "9b534e4318b7ebc3c961a839f13eb18b1833f441"
},
{
"model": "Qwen3-30B-A3B",
"model_id": null,
"params_b": 30.5,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-30B-A3B-GPTQ-Int4)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.841,
"quant_score": 0.835,
"retention": 0.9928656361474435,
"baseline_quant": "bf16 (unquantized Qwen3-30B-A3B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-30B-A3B-GPTQ-Int4/blob/9b534e4318b7ebc3c961a839f13eb18b1833f441/README.md#L328",
"source_id": "quant-qwen-hf-qwen3-30b-a3b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 328,
"raw_line": "| Non-Thinking | GPTQ-int4 | 57.2 | 50.4 | 83.5 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "83.5",
"header_line": 323
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-30B-A3B-GPTQ-Int4/README.md",
"sha256": "f6f8a413732e9218d75bb80cde4928d216f3e72d02cefb9fb46b5f626f1240fa",
"line": 327,
"raw_line": "| Non-Thinking | bf16 | 59.4 | 54.8 | 84.1 | - |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "84.1"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-30B-A3B-GPTQ-Int4 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim. No exact catalogue id: Qwen3-30B-A3B (April 2025 hybrid checkpoint) is not in data/models.json; the 2507 Instruct/Thinking checkpoints are different models.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-30B-A3B-GPTQ-Int4@9b534e4:Performance",
"artifact": "Qwen/Qwen3-30B-A3B-GPTQ-Int4",
"artifact_revision": "9b534e4318b7ebc3c961a839f13eb18b1833f441"
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-235B-A22B-GPTQ-Int4)",
"metric": "LiveBench 2024-11-25, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.771,
"quant_score": 0.751,
"retention": 0.9740596627756161,
"baseline_quant": "bf16 (unquantized Qwen3-235B-A22B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-GPTQ-Int4/blob/dff138951f3898772fc0d795a4b5b30f417577f4/README.md#L275",
"source_id": "quant-qwen-hf-qwen3-235b-a22b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 275,
"raw_line": "| Thinking | GPTQ-int4 | 75.1 | 71.9 | 92.0 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "75.1",
"header_line": 272
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 274,
"raw_line": "| Thinking | bf16 | 77.1 | 71.1 | 92.7 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "77.1"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-235B-A22B-GPTQ-Int4 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-235B-A22B-GPTQ-Int4@dff1389:Performance",
"artifact": "Qwen/Qwen3-235B-A22B-GPTQ-Int4",
"artifact_revision": "dff138951f3898772fc0d795a4b5b30f417577f4"
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-235B-A22B-GPTQ-Int4)",
"metric": "GPQA (subset not stated in card), thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.711,
"quant_score": 0.719,
"retention": 1.0112517580872014,
"baseline_quant": "bf16 (unquantized Qwen3-235B-A22B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-GPTQ-Int4/blob/dff138951f3898772fc0d795a4b5b30f417577f4/README.md#L275",
"source_id": "quant-qwen-hf-qwen3-235b-a22b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 275,
"raw_line": "| Thinking | GPTQ-int4 | 75.1 | 71.9 | 92.0 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "71.9",
"header_line": 272
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 274,
"raw_line": "| Thinking | bf16 | 77.1 | 71.1 | 92.7 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "71.1"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-235B-A22B-GPTQ-Int4 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-235B-A22B-GPTQ-Int4@dff1389:Performance",
"artifact": "Qwen/Qwen3-235B-A22B-GPTQ-Int4",
"artifact_revision": "dff138951f3898772fc0d795a4b5b30f417577f4"
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-235B-A22B-GPTQ-Int4)",
"metric": "MMLU-Redux, thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.927,
"quant_score": 0.92,
"retention": 0.9924487594390506,
"baseline_quant": "bf16 (unquantized Qwen3-235B-A22B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-GPTQ-Int4/blob/dff138951f3898772fc0d795a4b5b30f417577f4/README.md#L275",
"source_id": "quant-qwen-hf-qwen3-235b-a22b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 275,
"raw_line": "| Thinking | GPTQ-int4 | 75.1 | 71.9 | 92.0 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "92.0",
"header_line": 272
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 274,
"raw_line": "| Thinking | bf16 | 77.1 | 71.1 | 92.7 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "92.7"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-235B-A22B-GPTQ-Int4 card ('## Performance'). Thinking and non-thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-235B-A22B-GPTQ-Int4@dff1389:Performance",
"artifact": "Qwen/Qwen3-235B-A22B-GPTQ-Int4",
"artifact_revision": "dff138951f3898772fc0d795a4b5b30f417577f4"
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-235B-A22B-GPTQ-Int4)",
"metric": "LiveBench 2024-11-25, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.625,
"quant_score": 0.611,
"retention": 0.9776,
"baseline_quant": "bf16 (unquantized Qwen3-235B-A22B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-GPTQ-Int4/blob/dff138951f3898772fc0d795a4b5b30f417577f4/README.md#L277",
"source_id": "quant-qwen-hf-qwen3-235b-a22b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 277,
"raw_line": "| Non-Thinking | GPTQ-int4 | 61.1 | 62.8 | 89.0 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "61.1",
"header_line": 272
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 276,
"raw_line": "| Non-Thinking | bf16 | 62.5 | 62.9 | 89.2 |",
"column_header": "LiveBench 2024-11-25",
"column": 2,
"source_cell": "62.5"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-235B-A22B-GPTQ-Int4 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-235B-A22B-GPTQ-Int4@dff1389:Performance",
"artifact": "Qwen/Qwen3-235B-A22B-GPTQ-Int4",
"artifact_revision": "dff138951f3898772fc0d795a4b5b30f417577f4"
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-235B-A22B-GPTQ-Int4)",
"metric": "GPQA (subset not stated in card), non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.629,
"quant_score": 0.628,
"retention": 0.9984101748807631,
"baseline_quant": "bf16 (unquantized Qwen3-235B-A22B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-GPTQ-Int4/blob/dff138951f3898772fc0d795a4b5b30f417577f4/README.md#L277",
"source_id": "quant-qwen-hf-qwen3-235b-a22b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 277,
"raw_line": "| Non-Thinking | GPTQ-int4 | 61.1 | 62.8 | 89.0 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "62.8",
"header_line": 272
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 276,
"raw_line": "| Non-Thinking | bf16 | 62.5 | 62.9 | 89.2 |",
"column_header": "GPQA",
"column": 3,
"source_cell": "62.9"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-235B-A22B-GPTQ-Int4 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-235B-A22B-GPTQ-Int4@dff1389:Performance",
"artifact": "Qwen/Qwen3-235B-A22B-GPTQ-Int4",
"artifact_revision": "dff138951f3898772fc0d795a4b5b30f417577f4"
},
{
"model": "Qwen3-235B-A22B",
"model_id": "qwen3-235b-a22b",
"params_b": 235,
"quant": "GPTQ INT4 (w4, group_size 128, sym, desc_act=false; checkpoint Qwen/Qwen3-235B-A22B-GPTQ-Int4)",
"metric": "MMLU-Redux, non-thinking mode (Qwen-reported in quantized model card; harness, sampling and number of runs not stated)",
"full": 0.892,
"quant_score": 0.89,
"retention": 0.9977578475336323,
"baseline_quant": "bf16 (unquantized Qwen3-235B-A22B; 'bf16' row of the same card table, same evaluator)",
"source": "https://huggingface.co/Qwen/Qwen3-235B-A22B-GPTQ-Int4/blob/dff138951f3898772fc0d795a4b5b30f417577f4/README.md#L277",
"source_id": "quant-qwen-hf-qwen3-235b-a22b-gptq-int4",
"evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 277,
"raw_line": "| Non-Thinking | GPTQ-int4 | 61.1 | 62.8 | 89.0 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "89.0",
"header_line": 272
},
"baseline_evidence": {
"file": "speed-evidence/Qwen__Qwen3-235B-A22B-GPTQ-Int4/README.md",
"sha256": "c05fdde87e252b01dedac0db9d03e79239dd29851f1acade769e662b62dbc649",
"line": 276,
"raw_line": "| Non-Thinking | bf16 | 62.5 | 62.9 | 89.2 |",
"column_header": "MMLU-Redux",
"column": 4,
"source_cell": "89.2"
},
"transformation": "Published percentages / 100; retention = quant_score / bf16_score recomputed from the two table cells (unrounded)",
"note": "Qwen team's own quantized-vs-bf16 comparison in the Qwen/Qwen3-235B-A22B-GPTQ-Int4 card ('## Performance'). Non-Thinking and thinking modes are separate findings. Evaluation harness, sampling settings, number of runs and GPQA subset are not stated in the card; the recipe details (bits/group size) come from config.json at the same revision. Within-card relative evidence only; ratios above one are noise-level and are not an improvement claim.",
"aggregation": "single benchmark",
"study": "Qwen/Qwen3-235B-A22B-GPTQ-Int4@dff1389:Performance",
"artifact": "Qwen/Qwen3-235B-A22B-GPTQ-Int4",
"artifact_revision": "dff138951f3898772fc0d795a4b5b30f417577f4"
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "MLX 8-bit affine quantization (q8, group size not stated (mlx-lm default); mlx-community/Qwen3-4B-Instruct-2507-8bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.6405,
"quant_score": 0.6385,
"retention": 0.9968774395003903,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-4B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L32",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 32,
"raw_line": "q8 | 63.85 | 1606.573| 86.907 | 5.254 | mlx-community/Qwen3-4B-Instruct-2507-8bit"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 31,
"raw_line": "bf16 | 64.05 | 1780.63 | 52.47 | 9.02 | Qwen/Qwen3-4B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-4B-Instruct-2507",
"artifact": "mlx-community/Qwen3-4B-Instruct-2507-8bit",
"artifact_revision": null
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "MLX 6-bit affine quantization (q6, group size not stated (mlx-lm default); mlx-community/Qwen3-4B-Instruct-2507-6bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.6405,
"quant_score": 0.6353,
"retention": 0.9918813427010149,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-4B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L33",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 33,
"raw_line": "q6 | 63.53 | 1576.73 | 104.68 | 4.25 | mlx-community/Qwen3-4B-Instruct-2507-6bit"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 31,
"raw_line": "bf16 | 64.05 | 1780.63 | 52.47 | 9.02 | Qwen/Qwen3-4B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-4B-Instruct-2507",
"artifact": "mlx-community/Qwen3-4B-Instruct-2507-6bit",
"artifact_revision": null
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "MLX 5-bit affine quantization (q5 g32, group size 32; mlx-community/Qwen3-4B-Instruct-2507-5bit-g32)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.6405,
"quant_score": 0.6316,
"retention": 0.9861046057767371,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-4B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L34",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 34,
"raw_line": "q5 g32 | 63.16 | 1570.80 | 110.29 | 4.00 | mlx-community/Qwen3-4B-Instruct-2507-5bit-g32"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 31,
"raw_line": "bf16 | 64.05 | 1780.63 | 52.47 | 9.02 | Qwen/Qwen3-4B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-4B-Instruct-2507",
"artifact": "mlx-community/Qwen3-4B-Instruct-2507-5bit-g32",
"artifact_revision": null
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "MLX 5-bit affine quantization (q5, group size not stated (mlx-lm default); mlx-community/Qwen3-4B-Instruct-2507-5bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.6405,
"quant_score": 0.6238,
"retention": 0.9739266198282592,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-4B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L35",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 35,
"raw_line": "q5 | 62.38 | 1584.33 | 116.39 | 3.86 | mlx-community/Qwen3-4B-Instruct-2507-5bit"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 31,
"raw_line": "bf16 | 64.05 | 1780.63 | 52.47 | 9.02 | Qwen/Qwen3-4B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-4B-Instruct-2507",
"artifact": "mlx-community/Qwen3-4B-Instruct-2507-5bit",
"artifact_revision": null
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "MLX 4-bit affine quantization (q4 g32, group size 32; mlx-community/Qwen3-4B-Instruct-2507-4bit-g32)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.6405,
"quant_score": 0.6146,
"retention": 0.9595628415300548,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-4B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L36",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 36,
"raw_line": "q4 g32 | 61.46 | 1610.03 | 126.00 | 3.603 | mlx-community/Qwen3-4B-Instruct-2507-4bit-g32"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 31,
"raw_line": "bf16 | 64.05 | 1780.63 | 52.47 | 9.02 | Qwen/Qwen3-4B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-4B-Instruct-2507",
"artifact": "mlx-community/Qwen3-4B-Instruct-2507-4bit-g32",
"artifact_revision": null
},
{
"model": "Qwen3-4B-Instruct-2507",
"model_id": "qwen3-4b-instruct-2507",
"params_b": 4.02,
"quant": "MLX 4-bit affine quantization (q4, group size not stated (mlx-lm default); mlx-community/Qwen3-4B-Instruct-2507-4bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.6405,
"quant_score": 0.6072,
"retention": 0.9480093676814988,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-4B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L37",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 37,
"raw_line": "q4 | 60.72 | 1622.27 | 134.52 | 3.35 | mlx-community/Qwen3-4B-Instruct-2507-4bit"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 31,
"raw_line": "bf16 | 64.05 | 1780.63 | 52.47 | 9.02 | Qwen/Qwen3-4B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-4B-Instruct-2507",
"artifact": "mlx-community/Qwen3-4B-Instruct-2507-4bit",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "MLX 8-bit affine quantization (q8, group size not stated (mlx-lm default); mlx-community/Qwen3-30B-A3B-Instruct-2507-8bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.7262,
"quant_score": 0.7246,
"retention": 0.9977967502065548,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L52",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 52,
"raw_line": "q8 | 72.46 | 1719.47 | 83.16 | 33.46 | mlx-community/Qwen3-30B-A3B-Instruct-2507-8bit "
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 51,
"raw_line": "bf16 | 72.62 | :skull: | :skull: | :skull: | Qwen/Qwen3-30B-A3B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-30B-A3B-Instruct-2507",
"artifact": "mlx-community/Qwen3-30B-A3B-Instruct-2507-8bit",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "MLX 6-bit affine quantization (q6, group size not stated (mlx-lm default); mlx-community/Qwen3-30B-A3B-Instruct-2507-6bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.7262,
"quant_score": 0.7241,
"retention": 0.997108234646103,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L53",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 53,
"raw_line": "q6 | 72.41 | 1667.45 | 94.14 | 25.82 | mlx-community/Qwen3-30B-A3B-Instruct-2507-6bit"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 51,
"raw_line": "bf16 | 72.62 | :skull: | :skull: | :skull: | Qwen/Qwen3-30B-A3B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-30B-A3B-Instruct-2507",
"artifact": "mlx-community/Qwen3-30B-A3B-Instruct-2507-6bit",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "MLX 5-bit affine quantization (q5, group size not stated (mlx-lm default); mlx-community/Qwen3-30B-A3B-Instruct-2507-5bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.7262,
"quant_score": 0.7197,
"retention": 0.9910492977141284,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L54",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 54,
"raw_line": "q5 | 71.97 | 1664.24 | 101.00 |22.01 | mlx-community/Qwen3-30B-A3B-Instruct-2507-5bit"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 51,
"raw_line": "bf16 | 72.62 | :skull: | :skull: | :skull: | Qwen/Qwen3-30B-A3B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-30B-A3B-Instruct-2507",
"artifact": "mlx-community/Qwen3-30B-A3B-Instruct-2507-5bit",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "MLX 4-bit affine quantization (q4, group size not stated (mlx-lm default); mlx-community/Qwen3-30B-A3B-Instruct-2507-4bit)",
"metric": "MMLU-Pro (mlx_lm.evaluate --task mmlu_pro; lm-evaluation-harness task defaults, shots not stated in document)",
"full": 0.7262,
"quant_score": 0.7071,
"retention": 0.9736987055907463,
"baseline_quant": "bf16 (unquantized Qwen/Qwen3-30B-A3B-Instruct-2507, same mlx-lm benchmark table)",
"source": "https://github.com/ml-explore/mlx-lm/blob/87b7b583a697537aa68f47130b40884700b5f55f/mlx_lm/BENCHMARKS.md#L55",
"source_id": "speed-mlx-lm-benchmarks",
"evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 55,
"raw_line": "q4 | 70.71 | 1753.90 | 113.33 |18.20 | mlx-community/Qwen3-30B-A3B-Instruct-2507-4bit"
},
"baseline_evidence": {
"file": "speed-evidence/ml-explore__mlx-lm/mlx_lm/BENCHMARKS.md",
"sha256": "924047e9fa7e680490ceff2792c03390e7a13576a73280ef81cefa1380447218",
"line": 51,
"raw_line": "bf16 | 72.62 | :skull: | :skull: | :skull: | Qwen/Qwen3-30B-A3B-Instruct-2507"
},
"transformation": "MMLU Pro percentages / 100; retention = quant_score / bf16_score, recomputed and not rounded",
"note": "mlx-lm maintainers' own table: bf16 and quantized mlx-community checkpoints evaluated with the same documented mlx_lm.evaluate command. Single run; no uncertainty reported. Retention above 1 would be noise, not an improvement. Different recipe from GGUF K-quants.",
"aggregation": "single benchmark",
"study": "ml-explore/mlx-lm@87b7b58:BENCHMARKS.md:Qwen/Qwen3-30B-A3B-Instruct-2507",
"artifact": "mlx-community/Qwen3-30B-A3B-Instruct-2507-4bit",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q2_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.5535,
"retention": 0.8792692613185068,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L58",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 58,
"raw_line": "| **Optimized RTN** | q2_k_s | 0.5535(87.92%)| 0.6266(95.35%)|0.5901(93.35%)|0.6386(94.66%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q2_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q2_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6266,
"retention": 0.9535839293866992,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L58",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 58,
"raw_line": "| **Optimized RTN** | q2_k_s | 0.5535(87.92%)| 0.6266(95.35%)|0.5901(93.35%)|0.6386(94.66%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q2_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.5901,
"retention": 0.933407149636191,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L58",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 58,
"raw_line": "| **Optimized RTN** | q2_k_s | 0.5535(87.92%)| 0.6266(95.35%)|0.5901(93.35%)|0.6386(94.66%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q2_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "GGUF Q2_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6746,
"quant_score": 0.6386,
"retention": 0.9466350429884376,
"baseline_quant": "BF16 (unquantized Qwen3-30B-A3B-Instruct-2507, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L58",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 58,
"raw_line": "| **Optimized RTN** | q2_k_s | 0.5535(87.92%)| 0.6266(95.35%)|0.5901(93.35%)|0.6386(94.66%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-30B-A3B-Instruct-2507 GGUF q2_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q2_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.574,
"retention": 0.9118347895154885,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L59",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 59,
"raw_line": "| **AutoRound+alg_ext** |q2_k_s|0.5740(91.18%)|0.6349(96.62%)|0.5962(94.31%)|0.6460(95.77%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q2_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q2_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6349,
"retention": 0.96621518794704,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L59",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 59,
"raw_line": "| **AutoRound+alg_ext** |q2_k_s|0.5740(91.18%)|0.6349(96.62%)|0.5962(94.31%)|0.6460(95.77%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q2_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.5962,
"retention": 0.9430559949383106,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L59",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 59,
"raw_line": "| **AutoRound+alg_ext** |q2_k_s|0.5740(91.18%)|0.6349(96.62%)|0.5962(94.31%)|0.6460(95.77%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q2_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "GGUF Q2_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6746,
"quant_score": 0.646,
"retention": 0.9576045063741477,
"baseline_quant": "BF16 (unquantized Qwen3-30B-A3B-Instruct-2507, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L59",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 59,
"raw_line": "| **AutoRound+alg_ext** |q2_k_s|0.5740(91.18%)|0.6349(96.62%)|0.5962(94.31%)|0.6460(95.77%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-30B-A3B-Instruct-2507 GGUF q2_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q3_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.604,
"retention": 0.959491660047657,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L60",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 60,
"raw_line": "| **Optimized RTN** | q3_k_s | 0.6040(95.95%)|0.6382(97.12%)|0.6128(96.94%)|0.6598(97.82%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q3_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6382,
"retention": 0.971237254603561,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L60",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 60,
"raw_line": "| **Optimized RTN** | q3_k_s | 0.6040(95.95%)|0.6382(97.12%)|0.6128(96.94%)|0.6598(97.82%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q3_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.6128,
"retention": 0.969313508383423,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L60",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 60,
"raw_line": "| **Optimized RTN** | q3_k_s | 0.6040(95.95%)|0.6382(97.12%)|0.6128(96.94%)|0.6598(97.82%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q3_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "GGUF Q3_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6746,
"quant_score": 0.6598,
"retention": 0.97806107322858,
"baseline_quant": "BF16 (unquantized Qwen3-30B-A3B-Instruct-2507, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L60",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 60,
"raw_line": "| **Optimized RTN** | q3_k_s | 0.6040(95.95%)|0.6382(97.12%)|0.6128(96.94%)|0.6598(97.82%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-30B-A3B-Instruct-2507 GGUF q3_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q3_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.6081,
"retention": 0.9660047656870533,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L61",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 61,
"raw_line": "| **AutoRound+alg_ext** |q3_k_s|0.6081(96.59%)|0.6503(98.97%)|0.6252(98.89%)|0.6622(98.17%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q3_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6503,
"retention": 0.989651499010805,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L61",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 61,
"raw_line": "| **AutoRound+alg_ext** |q3_k_s|0.6081(96.59%)|0.6503(98.97%)|0.6252(98.89%)|0.6622(98.17%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q3_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.6252,
"retention": 0.9889275545713382,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L61",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 61,
"raw_line": "| **AutoRound+alg_ext** |q3_k_s|0.6081(96.59%)|0.6503(98.97%)|0.6252(98.89%)|0.6622(98.17%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q3_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "GGUF Q3_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6746,
"quant_score": 0.6622,
"retention": 0.9816187370293508,
"baseline_quant": "BF16 (unquantized Qwen3-30B-A3B-Instruct-2507, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L61",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 61,
"raw_line": "| **AutoRound+alg_ext** |q3_k_s|0.6081(96.59%)|0.6503(98.97%)|0.6252(98.89%)|0.6622(98.17%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-30B-A3B-Instruct-2507 GGUF q3_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q3_K_M produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.6083,
"retention": 0.9663224781572677,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L62",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 62,
"raw_line": "| **Optimized RTN** | q3_k_m |0.6083(96.63%) |0.6418(97.68%)|0.6194(97.97%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q3_k_m (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_M produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6418,
"retention": 0.9767158727743114,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L62",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 62,
"raw_line": "| **Optimized RTN** | q3_k_m |0.6083(96.63%) |0.6418(97.68%)|0.6194(97.97%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q3_k_m (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.6194,
"retention": 0.9797532426447326,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L62",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 62,
"raw_line": "| **Optimized RTN** | q3_k_m |0.6083(96.63%) |0.6418(97.68%)|0.6194(97.97%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q3_k_m (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q3_K_M produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.6127,
"retention": 0.9733121525019858,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L63",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 63,
"raw_line": "| **AutoRound+alg_ext** |q3_k_m|0.6127(97.33%)|0.6533(99.42%)|0.6197(98.02%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q3_k_m (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_M produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6533,
"retention": 0.9942170141530969,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L63",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 63,
"raw_line": "| **AutoRound+alg_ext** |q3_k_m|0.6127(97.33%)|0.6533(99.42%)|0.6197(98.02%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q3_k_m (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.6197,
"retention": 0.9802277760202468,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L63",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 63,
"raw_line": "| **AutoRound+alg_ext** |q3_k_m|0.6127(97.33%)|0.6533(99.42%)|0.6197(98.02%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q3_k_m (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q4_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.6228,
"retention": 0.9893566322478159,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L64",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 64,
"raw_line": "| **Optimized RTN** | q4_k_s | 0.6228(98.94%)|0.6560(99.83%)|0.6303(99.70%)|0.6762(100.24%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q4_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q4_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.656,
"retention": 0.9983259777811596,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L64",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 64,
"raw_line": "| **Optimized RTN** | q4_k_s | 0.6228(98.94%)|0.6560(99.83%)|0.6303(99.70%)|0.6762(100.24%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q4_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.6303,
"retention": 0.9969946219550775,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L64",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 64,
"raw_line": "| **Optimized RTN** | q4_k_s | 0.6228(98.94%)|0.6560(99.83%)|0.6303(99.70%)|0.6762(100.24%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q4_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "GGUF Q4_K_S produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6746,
"quant_score": 0.6762,
"retention": 1.0023717758671806,
"baseline_quant": "BF16 (unquantized Qwen3-30B-A3B-Instruct-2507, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L64",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 64,
"raw_line": "| **Optimized RTN** | q4_k_s | 0.6228(98.94%)|0.6560(99.83%)|0.6303(99.70%)|0.6762(100.24%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-30B-A3B-Instruct-2507 GGUF q4_k_s (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q4_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.6239,
"retention": 0.9911040508339953,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L65",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 65,
"raw_line": "| **AutoRound+alg_ext** |q4_k_s|0.6239(99.11%)|0.6605(100.51%)|0.6320(99.98%)|0.6777(100.46%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q4_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q4_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6605,
"retention": 1.0051742504945975,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L65",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 65,
"raw_line": "| **AutoRound+alg_ext** |q4_k_s|0.6239(99.11%)|0.6605(100.51%)|0.6320(99.98%)|0.6777(100.46%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q4_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.632,
"retention": 0.999683644416324,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L65",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 65,
"raw_line": "| **AutoRound+alg_ext** |q4_k_s|0.6239(99.11%)|0.6605(100.51%)|0.6320(99.98%)|0.6777(100.46%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q4_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-30B-A3B-Instruct-2507",
"model_id": "qwen3-30b-a3b-instruct-2507",
"params_b": 30.5,
"quant": "GGUF Q4_K_S produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6746,
"quant_score": 0.6777,
"retention": 1.0045953157426624,
"baseline_quant": "BF16 (unquantized Qwen3-30B-A3B-Instruct-2507, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L65",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 65,
"raw_line": "| **AutoRound+alg_ext** |q4_k_s|0.6239(99.11%)|0.6605(100.51%)|0.6320(99.98%)|0.6777(100.46%)|"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-30B-A3B-Instruct-2507 GGUF q4_k_s (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q4_K_M produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.6252,
"retention": 0.9931691818903893,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L66",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 66,
"raw_line": "| **Optimized RTN** | q4_k_m |0.6252(99.32%) |0.6558(99.80%)|0.6296(99.59%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q4_k_m (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q4_K_M produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6558,
"retention": 0.9980216101050069,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L66",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 66,
"raw_line": "| **Optimized RTN** | q4_k_m |0.6252(99.32%) |0.6558(99.80%)|0.6296(99.59%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q4_k_m (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M produced by AutoRound 'Optimized RTN' (--iters 0, no tuning), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.6296,
"retention": 0.9958873774122114,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L66",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 66,
"raw_line": "| **Optimized RTN** | q4_k_m |0.6252(99.32%) |0.6558(99.80%)|0.6296(99.59%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q4_k_m (Optimized RTN)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B (as labelled; base checkpoint implied, Instruct not stated)",
"model_id": null,
"params_b": 8,
"quant": "GGUF Q4_K_M produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6295,
"quant_score": 0.6257,
"retention": 0.9939634630659254,
"baseline_quant": "BF16 (unquantized Llama-3.1-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L67",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 67,
"raw_line": "| **AutoRound+alg_ext** |q4_k_m|0.6257(99.40%)|0.6575(100.06%)|0.6340(100.29%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Llama-3.1-8B GGUF q4_k_m (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q4_K_M produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6571,
"quant_score": 0.6575,
"retention": 1.0006087353523054,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L67",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 67,
"raw_line": "| **AutoRound+alg_ext** |q4_k_m|0.6257(99.40%)|0.6575(100.06%)|0.6340(100.29%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen2.5-7B-Instruct GGUF q4_k_m (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M produced by AutoRound + alg_ext (tuned), auto-round <=0.12.3",
"metric": "Average accuracy of 10 lm-eval tasks (lambada_openai, hellaswag, piqa, winogrande, truthfulqa_mc1, openbookqa, boolq, arc_easy, arc_challenge, mmlu); lm-eval defaults",
"full": 0.6322,
"quant_score": 0.634,
"retention": 1.0028472002530846,
"baseline_quant": "BF16 (unquantized Qwen3-8b, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L67",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 67,
"raw_line": "| **AutoRound+alg_ext** |q4_k_m|0.6257(99.40%)|0.6575(100.06%)|0.6340(100.29%)||"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 57,
"raw_line": "|**BF16** | - |0.6295(100%)|0.6571(100%) |0.6322(100%)|0.6746(100%) |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V<=0.12.3'). GGUF files produced by AutoRound's exporter, not llama.cpp's llama-quantize/imatrix; same K-quant type name does not imply the same file as unsloth/bartowski/Ollama builds. Llama models run with add_bos_token and a modified RMSNorm kernel decorator per the doc header. lm-eval version for this section not stated. Printed percentage in the cell was not used.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V<=0.12.3",
"artifact": "Qwen3-8b GGUF q4_k_m (AutoRound+alg_ext)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_S (row label 'q2ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "Average accuracy of the 10 listed lm-eval tasks (lm-eval 0.4.9.1, transformers 4.57.6)",
"full": 0.6329,
"quant_score": 0.606,
"retention": 0.9574972349502291,
"baseline_quant": "BF16 (unquantized Qwen3-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L17",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 17,
"raw_line": "| **q2ks** | 0.6060 | 0.5111 | 0.8026 | 0.8462 | 0.5107 | 0.6266 | 0.6736 | 0.3160 | 0.7492 | 0.3329 | 0.6906 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 15,
"raw_line": "| **bf16** | 0.6329 | 0.5563 | 0.8354 | 0.8670 | 0.5717 | 0.6439 | 0.7292 | 0.3160 | 0.7671 | 0.3647 | 0.6780 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen3-8B",
"artifact": "Qwen3-8B GGUF 'q2ks'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_S (row label 'q2ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "MMLU (lm-eval 0.4.9.1 default mmlu task, transformers 4.57.6)",
"full": 0.7292,
"quant_score": 0.6736,
"retention": 0.9237520570488207,
"baseline_quant": "BF16 (unquantized Qwen3-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L17",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 17,
"raw_line": "| **q2ks** | 0.6060 | 0.5111 | 0.8026 | 0.8462 | 0.5107 | 0.6266 | 0.6736 | 0.3160 | 0.7492 | 0.3329 | 0.6906 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 15,
"raw_line": "| **bf16** | 0.6329 | 0.5563 | 0.8354 | 0.8670 | 0.5717 | 0.6439 | 0.7292 | 0.3160 | 0.7671 | 0.3647 | 0.6780 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen3-8B",
"artifact": "Qwen3-8B GGUF 'q2ks'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_S (row label 'q3ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "Average accuracy of the 10 listed lm-eval tasks (lm-eval 0.4.9.1, transformers 4.57.6)",
"full": 0.6329,
"quant_score": 0.6287,
"retention": 0.9933638805498499,
"baseline_quant": "BF16 (unquantized Qwen3-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L19",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 19,
"raw_line": "| **q3ks** | 0.6287 | 0.5580 | 0.8316 | 0.8566 | 0.5516 | 0.6385 | 0.7090 | 0.3300 | 0.7661 | 0.3488 | 0.6969 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 15,
"raw_line": "| **bf16** | 0.6329 | 0.5563 | 0.8354 | 0.8670 | 0.5717 | 0.6439 | 0.7292 | 0.3160 | 0.7671 | 0.3647 | 0.6780 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen3-8B",
"artifact": "Qwen3-8B GGUF 'q3ks'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_S (row label 'q3ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "MMLU (lm-eval 0.4.9.1 default mmlu task, transformers 4.57.6)",
"full": 0.7292,
"quant_score": 0.709,
"retention": 0.9722984092155788,
"baseline_quant": "BF16 (unquantized Qwen3-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L19",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 19,
"raw_line": "| **q3ks** | 0.6287 | 0.5580 | 0.8316 | 0.8566 | 0.5516 | 0.6385 | 0.7090 | 0.3300 | 0.7661 | 0.3488 | 0.6969 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 15,
"raw_line": "| **bf16** | 0.6329 | 0.5563 | 0.8354 | 0.8670 | 0.5717 | 0.6439 | 0.7292 | 0.3160 | 0.7671 | 0.3647 | 0.6780 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen3-8B",
"artifact": "Qwen3-8B GGUF 'q3ks'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (row label 'q4km'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "Average accuracy of the 10 listed lm-eval tasks (lm-eval 0.4.9.1, transformers 4.57.6)",
"full": 0.6329,
"quant_score": 0.6353,
"retention": 1.0037920682572286,
"baseline_quant": "BF16 (unquantized Qwen3-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L21",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 21,
"raw_line": "| **q4km** | 0.6353 | 0.5606 | 0.8388 | 0.8673 | 0.5695 | 0.6392 | 0.7285 | 0.3180 | 0.7677 | 0.3770 | 0.6867 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 15,
"raw_line": "| **bf16** | 0.6329 | 0.5563 | 0.8354 | 0.8670 | 0.5717 | 0.6439 | 0.7292 | 0.3160 | 0.7671 | 0.3647 | 0.6780 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen3-8B",
"artifact": "Qwen3-8B GGUF 'q4km'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (row label 'q4km'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "MMLU (lm-eval 0.4.9.1 default mmlu task, transformers 4.57.6)",
"full": 0.7292,
"quant_score": 0.7285,
"retention": 0.9990400438837083,
"baseline_quant": "BF16 (unquantized Qwen3-8B, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L21",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 21,
"raw_line": "| **q4km** | 0.6353 | 0.5606 | 0.8388 | 0.8673 | 0.5695 | 0.6392 | 0.7285 | 0.3180 | 0.7677 | 0.3770 | 0.6867 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 15,
"raw_line": "| **bf16** | 0.6329 | 0.5563 | 0.8354 | 0.8670 | 0.5717 | 0.6439 | 0.7292 | 0.3160 | 0.7671 | 0.3647 | 0.6780 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen3-8B",
"artifact": "Qwen3-8B GGUF 'q4km'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q2_K_S (row label 'q2ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "Average accuracy of the 10 listed lm-eval tasks (lm-eval 0.4.9.1, transformers 4.57.6)",
"full": 0.6574,
"quant_score": 0.6388,
"retention": 0.971706723456039,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L31",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 31,
"raw_line": "| **q2ks** | 0.6388 | 0.5205 | 0.8127 | 0.8636 | 0.5652 | 0.6975 | 0.6741 | 0.3300 | 0.7688 | 0.4431 | 0.7127 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q2ks'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q2_K_S (row label 'q2ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "MMLU (lm-eval 0.4.9.1 default mmlu task, transformers 4.57.6)",
"full": 0.7161,
"quant_score": 0.6741,
"retention": 0.9413489736070382,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L31",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 31,
"raw_line": "| **q2ks** | 0.6388 | 0.5205 | 0.8127 | 0.8636 | 0.5652 | 0.6975 | 0.6741 | 0.3300 | 0.7688 | 0.4431 | 0.7127 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q2ks'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_S (row label 'q3ks-opt-rtn'; AutoRound 'Optimized RTN'; auto-round >0.12.3)",
"metric": "Average accuracy of the 10 listed lm-eval tasks (lm-eval 0.4.9.1, transformers 4.57.6)",
"full": 0.6574,
"quant_score": 0.638,
"retention": 0.9704898083358686,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L32",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 32,
"raw_line": "| **q3ks-opt-rtn** | 0.6380 | 0.5009 | 0.7870 | 0.8569 | 0.5964 | 0.7054 | 0.6908 | 0.3400 | 0.7688 | 0.4578 | 0.6756 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q3ks-opt-rtn'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_S (row label 'q3ks-opt-rtn'; AutoRound 'Optimized RTN'; auto-round >0.12.3)",
"metric": "MMLU (lm-eval 0.4.9.1 default mmlu task, transformers 4.57.6)",
"full": 0.7161,
"quant_score": 0.6908,
"retention": 0.9646697388632872,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L32",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 32,
"raw_line": "| **q3ks-opt-rtn** | 0.6380 | 0.5009 | 0.7870 | 0.8569 | 0.5964 | 0.7054 | 0.6908 | 0.3400 | 0.7688 | 0.4578 | 0.6756 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q3ks-opt-rtn'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_S (row label 'q3ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "Average accuracy of the 10 listed lm-eval tasks (lm-eval 0.4.9.1, transformers 4.57.6)",
"full": 0.6574,
"quant_score": 0.6482,
"retention": 0.9860054761180408,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L33",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 33,
"raw_line": "| **q3ks** | 0.6482 | 0.5358 | 0.8123 | 0.8575 | 0.6024 | 0.6986 | 0.7029 | 0.3340 | 0.7769 | 0.4663 | 0.6953 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q3ks'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q3_K_S (row label 'q3ks'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "MMLU (lm-eval 0.4.9.1 default mmlu task, transformers 4.57.6)",
"full": 0.7161,
"quant_score": 0.7029,
"retention": 0.9815668202764977,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L33",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 33,
"raw_line": "| **q3ks** | 0.6482 | 0.5358 | 0.8123 | 0.8575 | 0.6024 | 0.6986 | 0.7029 | 0.3340 | 0.7769 | 0.4663 | 0.6953 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q3ks'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q4_K_M (row label 'q4km'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "Average accuracy of the 10 listed lm-eval tasks (lm-eval 0.4.9.1, transformers 4.57.6)",
"full": 0.6574,
"quant_score": 0.6585,
"retention": 1.0016732582902343,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L35",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 35,
"raw_line": "| **q4km** | 0.6585 | 0.5478 | 0.8190 | 0.8630 | 0.6204 | 0.7004 | 0.7141 | 0.3420 | 0.7916 | 0.4774 | 0.7088 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q4km'",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GGUF Q4_K_M (row label 'q4km'; AutoRound default GGUF path (tuning mode not restated in the row label); auto-round >0.12.3)",
"metric": "MMLU (lm-eval 0.4.9.1 default mmlu task, transformers 4.57.6)",
"full": 0.7161,
"quant_score": 0.7141,
"retention": 0.9972070939812875,
"baseline_quant": "BF16 (unquantized Qwen2.5-7B-Instruct, same table)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/gguf_alg_ext_acc.md#L35",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 35,
"raw_line": "| **q4km** | 0.6585 | 0.5478 | 0.8190 | 0.8630 | 0.6204 | 0.7004 | 0.7141 | 0.3420 | 0.7916 | 0.4774 | 0.7088 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/gguf_alg_ext_acc.md",
"sha256": "5b4e56630a65c41eb737a364a40bb05d07bc589f1eff0c6d79d41c82b1e9ecab",
"line": 29,
"raw_line": "| **BF16** | 0.6574 | 0.5282 | 0.8144 | 0.8642 | 0.6201 | 0.6953 | 0.7161 | 0.3480 | 0.7938 | 0.4823 | 0.7119 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers' own evaluation (docs/gguf_alg_ext_acc.md, 'Results of V>0.12.3'). GGUF produced by AutoRound, not llama-quantize/imatrix; not asserted identical to community or Ollama files of the same type. The 'opt-rtn (Nbit)' rows are excluded because they do not name the GGUF type.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:gguf_alg_ext_acc:V>0.12.3:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct GGUF 'q4km'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K (unsloth/Qwen3-8B-GGUF file as labelled; actual 3.20 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6075,
"retention": 0.9603224786595005,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L45",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 45,
"raw_line": "| unsloth | Q2_K | 3.20 | 0.6075 | 0.5145 | 0.8051 | 0.8526 | 0.5309 | 0.6255 | 0.3166 | 0.7530 | 0.3452 | 0.6590 | 0.6729 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K (unsloth/Qwen3-8B-GGUF file as labelled; actual 3.20 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.6729,
"retention": 0.9216545678674155,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L45",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 45,
"raw_line": "| unsloth | Q2_K | 3.20 | 0.6075 | 0.5145 | 0.8051 | 0.8526 | 0.5309 | 0.6255 | 0.3166 | 0.7530 | 0.3452 | 0.6590 | 0.6729 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K (bartowski/Qwen3-8B-GGUF file as labelled; actual 3.20 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6071,
"retention": 0.9596901675624406,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L46",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 46,
"raw_line": "| bartowski | Q2_K | 3.20 | 0.6071 | 0.5128 | 0.8018 | 0.8511 | 0.5312 | 0.6257 | 0.3100 | 0.7568 | 0.3366 | 0.6677 | 0.6772 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q2_K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K (bartowski/Qwen3-8B-GGUF file as labelled; actual 3.20 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.6772,
"retention": 0.9275441720312286,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L46",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 46,
"raw_line": "| bartowski | Q2_K | 3.20 | 0.6071 | 0.5128 | 0.8018 | 0.8511 | 0.5312 | 0.6257 | 0.3100 | 0.7568 | 0.3366 | 0.6677 | 0.6772 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q2_K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4]' (actual 3.23 bits/weight); no AutoRound tuning per line 35",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6153,
"retention": 0.9726525450521655,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L47",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 47,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] | 3.23 | 0.6153 | 0.5239 | 0.8102 | 0.8615 | 0.5242 | 0.6598 | 0.3140 | 0.7503 | 0.3390 | 0.6756 | 0.6943 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4]' (actual 3.23 bits/weight); no AutoRound tuning per line 35",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.6943,
"retention": 0.950965621147788,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L47",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 47,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] | 3.23 | 0.6153 | 0.5239 | 0.8102 | 0.8615 | 0.5242 | 0.6598 | 0.3140 | 0.7503 | 0.3390 | 0.6756 | 0.6943 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6]' (actual 3.24 bits/weight); no AutoRound tuning per line 35",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6074,
"retention": 0.9601644008852356,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L48",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 48,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] | 3.24 | 0.6074 | 0.5034 | 0.8131 | 0.8508 | 0.5109 | 0.6571 | 0.3020 | 0.7470 | 0.3317 | 0.6685 | 0.6897 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4,q6]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6]' (actual 3.24 bits/weight); no AutoRound tuning per line 35",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.6897,
"retention": 0.9446651143678948,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L48",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 48,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] | 3.24 | 0.6074 | 0.5034 | 0.8131 | 0.8508 | 0.5109 | 0.6571 | 0.3020 | 0.7470 | 0.3317 | 0.6685 | 0.6897 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4,q6]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4] lm_head Q6K' (actual 3.25 bits/weight); no AutoRound tuning per line 35",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6093,
"retention": 0.9631678785962692,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L49",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 49,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] lm_head Q6K | 3.25 | 0.6093 | 0.5060 | 0.8093 | 0.8538 | 0.5119 | 0.6598 | 0.3040 | 0.7459 | 0.3256 | 0.6835 | 0.6929 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4] lm_head Q6K'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4] lm_head Q6K' (actual 3.25 bits/weight); no AutoRound tuning per line 35",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.6929,
"retention": 0.9490480756060813,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L49",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 49,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] lm_head Q6K | 3.25 | 0.6093 | 0.5060 | 0.8093 | 0.8538 | 0.5119 | 0.6598 | 0.3040 | 0.7459 | 0.3256 | 0.6835 | 0.6929 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4] lm_head Q6K'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6] iters 200' (actual 3.25 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6133,
"retention": 0.9694909895668667,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L50",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 50,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] iters 200 | 3.25 | 0.6133 | 0.5060 | 0.8136 | 0.8541 | 0.5234 | 0.6565 | 0.3040 | 0.7606 | 0.3403 | 0.6827 | 0.6919 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4,q6] iters 200'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6] iters 200' (actual 3.25 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.6919,
"retention": 0.9476784002191481,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L50",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 50,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] iters 200 | 3.25 | 0.6133 | 0.5060 | 0.8136 | 0.8541 | 0.5234 | 0.6565 | 0.3040 | 0.7606 | 0.3403 | 0.6827 | 0.6919 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '2.5b mixed[q2,q3,q4,q6] iters 200'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_L (unsloth/Qwen3-8B-GGUF file as labelled; actual 3.34 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6078,
"retention": 0.9607967119822952,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L51",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 51,
"raw_line": "| unsloth | Q2_K_L | 3.34 | 0.6078 | 0.5128 | 0.8068 | 0.8529 | 0.5295 | 0.6276 | 0.3189 | 0.7546 | 0.3427 | 0.6567 | 0.6750 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K_L",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_L (unsloth/Qwen3-8B-GGUF file as labelled; actual 3.34 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.675,
"retention": 0.9245308861799755,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L51",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 51,
"raw_line": "| unsloth | Q2_K_L | 3.34 | 0.6078 | 0.5128 | 0.8068 | 0.8529 | 0.5295 | 0.6276 | 0.3189 | 0.7546 | 0.3427 | 0.6567 | 0.6750 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K_L",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_L (bartowski/Qwen3-8B-GGUF file as labelled; actual 3.79 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6076,
"retention": 0.9604805564337654,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L52",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 52,
"raw_line": "| bartowski | Q2_K_L | 3.79 | 0.6076 | 0.5162 | 0.8018 | 0.8449 | 0.5306 | 0.6266 | 0.3120 | 0.7606 | 0.3305 | 0.6717 | 0.6806 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q2_K_L",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_L (bartowski/Qwen3-8B-GGUF file as labelled; actual 3.79 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.6806,
"retention": 0.9322010683468018,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L52",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 52,
"raw_line": "| bartowski | Q2_K_L | 3.79 | 0.6076 | 0.5162 | 0.8018 | 0.8449 | 0.5306 | 0.6266 | 0.3120 | 0.7606 | 0.3305 | 0.6717 | 0.6806 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q2_K_L",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M (unsloth/Qwen3-8B-GGUF file as labelled; actual 4.02 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.626,
"retention": 0.989566866898514,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L53",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 53,
"raw_line": "| unsloth | Q3_K_M | 4.02 | 0.6260 | 0.5503 | 0.8270 | 0.8596 | 0.5589 | 0.6301 | 0.3120 | 0.7655 | 0.3770 | 0.6677 | 0.7115 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M (unsloth/Qwen3-8B-GGUF file as labelled; actual 4.02 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7115,
"retention": 0.9745240378030408,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L53",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 53,
"raw_line": "| unsloth | Q3_K_M | 4.02 | 0.6260 | 0.5503 | 0.8270 | 0.8596 | 0.5589 | 0.6301 | 0.3120 | 0.7655 | 0.3770 | 0.6677 | 0.7115 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M (bartowski/Qwen3-8B-GGUF file as labelled; actual 4.02 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6246,
"retention": 0.987353778058805,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L54",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 54,
"raw_line": "| bartowski | Q3_K_M | 4.02 | 0.6246 | 0.5478 | 0.8304 | 0.8606 | 0.5554 | 0.6321 | 0.3200 | 0.7661 | 0.3647 | 0.6575 | 0.7118 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M (bartowski/Qwen3-8B-GGUF file as labelled; actual 4.02 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7118,
"retention": 0.9749349404191207,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L54",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 54,
"raw_line": "| bartowski | Q3_K_M | 4.02 | 0.6246 | 0.5478 | 0.8304 | 0.8606 | 0.5554 | 0.6321 | 0.3200 | 0.7661 | 0.3647 | 0.6575 | 0.7118 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4]' (actual 4.17 bits/weight); no AutoRound tuning per line 35",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6284,
"retention": 0.9933607334808724,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L55",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 55,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] | 4.17 | 0.6284 | 0.5478 | 0.8300 | 0.8648 | 0.5651 | 0.6429 | 0.3100 | 0.7612 | 0.3574 | 0.6819 | 0.7225 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4]' (actual 4.17 bits/weight); no AutoRound tuning per line 35",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7225,
"retention": 0.989590467059307,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L55",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 55,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] | 4.17 | 0.6284 | 0.5478 | 0.8300 | 0.8648 | 0.5651 | 0.6429 | 0.3100 | 0.7612 | 0.3574 | 0.6819 | 0.7225 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4] lm_head Q6K' (actual 4.19 bits/weight); no AutoRound tuning per line 35",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6251,
"retention": 0.9881441669301295,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L56",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 56,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] lm_head Q6K | 4.19 | 0.6251 | 0.5384 | 0.8338 | 0.8609 | 0.5581 | 0.6354 | 0.3160 | 0.7612 | 0.3574 | 0.6725 | 0.7175 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4] lm_head Q6K'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4] lm_head Q6K' (actual 4.19 bits/weight); no AutoRound tuning per line 35",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7175,
"retention": 0.9827420901246405,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L56",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 56,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] lm_head Q6K | 4.19 | 0.6251 | 0.5384 | 0.8338 | 0.8609 | 0.5581 | 0.6354 | 0.3160 | 0.7612 | 0.3574 | 0.6725 | 0.7175 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4] lm_head Q6K'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6]' (actual 4.19 bits/weight); no AutoRound tuning per line 35",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6246,
"retention": 0.987353778058805,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L57",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 57,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] | 4.19 | 0.6246 | 0.5401 | 0.8259 | 0.8624 | 0.5582 | 0.6387 | 0.3080 | 0.7628 | 0.3586 | 0.6725 | 0.7191 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4,q6]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6]' (actual 4.19 bits/weight); no AutoRound tuning per line 35",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7191,
"retention": 0.9849335707437337,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L57",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 57,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] | 4.19 | 0.6246 | 0.5401 | 0.8259 | 0.8624 | 0.5582 | 0.6387 | 0.3080 | 0.7628 | 0.3586 | 0.6725 | 0.7191 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4,q6]'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6] iters 200' (actual 4.19 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6318,
"retention": 0.9987353778058805,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L58",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 58,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] iters 200 | 4.19 | 0.6318 | 0.5520 | 0.8346 | 0.8676 | 0.5624 | 0.6404 | 0.3160 | 0.7682 | 0.3696 | 0.6890 | 0.7183 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4,q6] iters 200'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6] iters 200' (actual 4.19 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7183,
"retention": 0.9838378304341872,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L58",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 58,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] iters 200 | 4.19 | 0.6318 | 0.5520 | 0.8346 | 0.8676 | 0.5624 | 0.6404 | 0.3160 | 0.7682 | 0.3696 | 0.6890 | 0.7183 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "AutoRound-generated Qwen3-8B GGUF '3.5b mixed[q2,q3,q4,q6] iters 200'",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (unsloth/Qwen3-8B-GGUF file as labelled; actual 4.90 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6308,
"retention": 0.9971546000632311,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L59",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 59,
"raw_line": "| unsloth | Q4_K_M | 4.90 | 0.6308 | 0.5520 | 0.8329 | 0.8691 | 0.5685 | 0.6437 | 0.3180 | 0.7568 | 0.3623 | 0.6803 | 0.7248 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (unsloth/Qwen3-8B-GGUF file as labelled; actual 4.90 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7248,
"retention": 0.9927407204492535,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L59",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 59,
"raw_line": "| unsloth | Q4_K_M | 4.90 | 0.6308 | 0.5520 | 0.8329 | 0.8691 | 0.5685 | 0.6437 | 0.3180 | 0.7568 | 0.3623 | 0.6803 | 0.7248 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (bartowski/Qwen3-8B-GGUF file as labelled; actual 4.90 bits/weight)",
"metric": "Average accuracy of 10 lm-eval tasks (arc_challenge, arc_easy, boolq, hellaswag, lambada_openai, mmlu, openbookqa, piqa, truthfulqa_mc1, winogrande); lm_eval --model hf with gguf_file",
"full": 0.6326,
"quant_score": 0.6301,
"retention": 0.9960480556433764,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L60",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 60,
"raw_line": "| bartowski | Q4_K_M | 4.90 | 0.6301 | 0.5546 | 0.8359 | 0.8654 | 0.5655 | 0.6443 | 0.3120 | 0.7612 | 0.3599 | 0.6748 | 0.7269 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "suite average of 10 tasks",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (bartowski/Qwen3-8B-GGUF file as labelled; actual 4.90 bits/weight)",
"metric": "MMLU (lm-eval default mmlu task; lm_eval --model hf with gguf_file)",
"full": 0.7301,
"quant_score": 0.7269,
"retention": 0.9956170387618135,
"baseline_quant": "F16 GGUF (same table, line 44)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L60",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 60,
"raw_line": "| bartowski | Q4_K_M | 4.90 | 0.6301 | 0.5546 | 0.8359 | 0.8654 | 0.5655 | 0.6443 | 0.3120 | 0.7612 | 0.3599 | 0.6748 | 0.7269 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 44,
"raw_line": "| f16 | f16 | 16 | 0.6326 | 0.5546 | 0.8333 | 0.8664 | 0.5717 | 0.6416 | 0.3140 | 0.7671 | 0.3672 | 0.6796 | 0.7301 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers evaluated each GGUF file with lm-eval's Hugging Face backend loading gguf_file (transformers dequantizes GGUF weights, so quantization error is retained but llama.cpp kernels are not exercised). Single run; lm-eval/transformers versions for this table not stated.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:lm-eval:Qwen3-8B",
"artifact": "bartowski Qwen3-8B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K (unsloth file as labelled; actual 3.20 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.3131,
"retention": 0.6812445604873802,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L89",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 89,
"raw_line": "| unsloth | Q2_K | 3.20 | 0.3131 | 0.726 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K (unsloth file as labelled; actual 3.20 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.726,
"retention": 0.9097744360902255,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L89",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 89,
"raw_line": "| unsloth | Q2_K | 3.20 | 0.3131 | 0.726 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4]' (actual 3.23 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.3636,
"retention": 0.7911227154046997,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L90",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 90,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] | 3.23 | 0.3636 | 0.706 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4]' (actual 3.23 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.706,
"retention": 0.8847117794486214,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L90",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 90,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] | 3.23 | 0.3636 | 0.706 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6]' (actual 3.24 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.3687,
"retention": 0.8022193211488251,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L91",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 91,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] | 3.24 | 0.3687 | 0.698 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6]' (actual 3.24 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.698,
"retention": 0.8746867167919798,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L91",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 91,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] | 3.24 | 0.3687 | 0.698 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4] lm_head Q6K' (actual 3.25 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.3485,
"retention": 0.7582680591818972,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L92",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 92,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] lm_head Q6K | 3.25 | 0.3485 | 0.714 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4] lm_head Q6K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4] lm_head Q6K' (actual 3.25 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.714,
"retention": 0.894736842105263,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L92",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 92,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4] lm_head Q6K | 3.25 | 0.3485 | 0.714 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4] lm_head Q6K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6] iters 200' (actual 3.25 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.399,
"retention": 0.8681462140992168,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L93",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 93,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] iters 200 | 3.25 | 0.3990 | 0.700 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4,q6] iters 200",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5b mixed[q2,q3,q4,q6] iters 200' (actual 3.25 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.7,
"retention": 0.8771929824561403,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L93",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 93,
"raw_line": "| auto_round | 2.5b mixed[q2,q3,q4,q6] iters 200 | 3.25 | 0.3990 | 0.700 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 2.5b mixed[q2,q3,q4,q6] iters 200",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_L (unsloth file as labelled; actual 3.34 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.3788,
"retention": 0.824194952132289,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L94",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 94,
"raw_line": "| unsloth | Q2_K_L | 3.34 | 0.3788 | 0.732 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K_L",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K_L (unsloth file as labelled; actual 3.34 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.732,
"retention": 0.9172932330827067,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L94",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 94,
"raw_line": "| unsloth | Q2_K_L | 3.34 | 0.3788 | 0.732 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q2_K_L",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M (unsloth file as labelled; actual 4.02 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.4192,
"retention": 0.9120974760661444,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L95",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 95,
"raw_line": "| unsloth | Q3_K_M | 4.02 | 0.4192 | 0.778 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q3_K_M (unsloth file as labelled; actual 4.02 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.778,
"retention": 0.974937343358396,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L95",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 95,
"raw_line": "| unsloth | Q3_K_M | 4.02 | 0.4192 | 0.778 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4]' (actual 4.17 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.4495,
"retention": 0.9780243690165361,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L96",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 96,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] | 4.17 | 0.4495 | 0.816 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4]' (actual 4.17 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.816,
"retention": 1.0225563909774436,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L96",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 96,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] | 4.17 | 0.4495 | 0.816 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4] lm_head Q6K' (actual 4.19 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.4495,
"retention": 0.9780243690165361,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L97",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 97,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] lm_head Q6K | 4.19 | 0.4495 | 0.788 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4] lm_head Q6K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4] lm_head Q6K' (actual 4.19 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.788,
"retention": 0.9874686716791979,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L97",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 97,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4] lm_head Q6K | 4.19 | 0.4495 | 0.788 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4] lm_head Q6K",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6]' (actual 4.19 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.4444,
"retention": 0.9669277632724108,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L98",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 98,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] | 4.19 | 0.4444 | 0.814 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6]' (actual 4.19 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.814,
"retention": 1.020050125313283,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L98",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 98,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] | 4.19 | 0.4444 | 0.814 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6] iters 200' (actual 4.19 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.5051,
"retention": 1.0989991296779809,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L99",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 99,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] iters 200 | 4.19 | 0.5051 | 0.788 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4,q6] iters 200",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5b mixed[q2,q3,q4,q6] iters 200' (actual 4.19 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.788,
"retention": 0.9874686716791979,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L99",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 99,
"raw_line": "| auto_round | 3.5b mixed[q2,q3,q4,q6] iters 200 | 4.19 | 0.5051 | 0.788 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "auto_round Qwen3-8B GGUF 3.5b mixed[q2,q3,q4,q6] iters 200",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (unsloth file as labelled; actual 4.90 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4596,
"quant_score": 0.4545,
"retention": 0.9889033942558747,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L100",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 100,
"raw_line": "| unsloth | Q4_K_M | 4.90 | 0.4545 | 0.792 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M (unsloth file as labelled; actual 4.90 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.798,
"quant_score": 0.792,
"retention": 0.9924812030075187,
"baseline_quant": "F16 GGUF (same table, line 88)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L100",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 100,
"raw_line": "| unsloth | Q4_K_M | 4.90 | 0.4545 | 0.792 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 88,
"raw_line": "| f16 | f16 | 16 | 0.4596 | 0.798 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3-8B",
"artifact": "unsloth Qwen3-8B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF UD-IQ2_M (unsloth file as labelled; actual 3.33 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.2273,
"retention": 0.48392591015541836,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L105",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 105,
"raw_line": "| unsloth | UD-IQ2_M | 3.33 | 0.2273 | 0.594 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF UD-IQ2_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF UD-IQ2_M (unsloth file as labelled; actual 3.33 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.594,
"retention": 0.7518987341772151,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L105",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 105,
"raw_line": "| unsloth | UD-IQ2_M | 3.33 | 0.2273 | 0.594 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF UD-IQ2_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5 bits mixed[q2,q3,q4,q6]' (actual 3.9 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.4242,
"retention": 0.903129657228018,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L106",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 106,
"raw_line": "| auto_round | 2.5 bits mixed[q2,q3,q4,q6] | 3.9 | 0.4242 | 0.72 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 2.5 bits mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5 bits mixed[q2,q3,q4,q6]' (actual 3.9 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.72,
"retention": 0.9113924050632911,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L106",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 106,
"raw_line": "| auto_round | 2.5 bits mixed[q2,q3,q4,q6] | 3.9 | 0.4242 | 0.72 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 2.5 bits mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5 bits mixed[q2,q3,q4,q6] tuning iters 200' (actual 3.9 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.3838,
"retention": 0.8171173089205875,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L107",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 107,
"raw_line": "| auto_round | 2.5 bits mixed[q2,q3,q4,q6] tuning iters 200 | 3.9 | 0.3838 | 0.754 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 2.5 bits mixed[q2,q3,q4,q6] tuning iters 200",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '2.5 bits mixed[q2,q3,q4,q6] tuning iters 200' (actual 3.9 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.754,
"retention": 0.9544303797468354,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L107",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 107,
"raw_line": "| auto_round | 2.5 bits mixed[q2,q3,q4,q6] tuning iters 200 | 3.9 | 0.3838 | 0.754 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 2.5 bits mixed[q2,q3,q4,q6] tuning iters 200",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q3_K_M (AutoRound-generated; actual 4.17 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.4042,
"retention": 0.8605492867787949,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L108",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 108,
"raw_line": "| auto_round | Q3_K_M | 4.17 | 0.4042 | 0.772 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q3_K_M (AutoRound-generated; actual 4.17 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.772,
"retention": 0.9772151898734177,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L108",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 108,
"raw_line": "| auto_round | Q3_K_M | 4.17 | 0.4042 | 0.772 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q3_K_M (unsloth file as labelled; actual 4.34 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.404,
"retention": 0.8601234830743028,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L109",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 109,
"raw_line": "| unsloth | Q3_K_M | 4.34 | 0.4040 | 0.776 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q3_K_M (unsloth file as labelled; actual 4.34 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.776,
"retention": 0.9822784810126582,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L109",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 109,
"raw_line": "| unsloth | Q3_K_M | 4.34 | 0.4040 | 0.776 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF Q3_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5 bits mixed[q2,q3,q4,q6]' (actual 4.86 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.4192,
"retention": 0.8924845646157121,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L110",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 110,
"raw_line": "| auto_round | 3.5 bits mixed[q2,q3,q4,q6] | 4.86 | 0.4192 | 0.804 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 3.5 bits mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5 bits mixed[q2,q3,q4,q6]' (actual 4.86 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.804,
"retention": 1.0177215189873419,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L110",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 110,
"raw_line": "| auto_round | 3.5 bits mixed[q2,q3,q4,q6] | 4.86 | 0.4192 | 0.804 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 3.5 bits mixed[q2,q3,q4,q6]",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5 bits mixed[q2,q3,q4,q6] tuning iters 200' (actual 4.86 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.4192,
"retention": 0.8924845646157121,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L111",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 111,
"raw_line": "| auto_round | 3.5 bits mixed[q2,q3,q4,q6] tuning iters 200 | 4.86 | 0.4192 | 0.786 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 3.5 bits mixed[q2,q3,q4,q6] tuning iters 200",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "AutoRound AutoScheme mixed-precision GGUF '3.5 bits mixed[q2,q3,q4,q6] tuning iters 200' (actual 4.86 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.786,
"retention": 0.9949367088607595,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L111",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 111,
"raw_line": "| auto_round | 3.5 bits mixed[q2,q3,q4,q6] tuning iters 200 | 4.86 | 0.4192 | 0.786 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF 3.5 bits mixed[q2,q3,q4,q6] tuning iters 200",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (AutoRound-generated; actual 5.12 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.4545,
"retention": 0.9676389184585906,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L112",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 112,
"raw_line": "| auto_round | Q4_K_M | 5.12 | 0.4545 | 0.792 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (AutoRound-generated; actual 5.12 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.792,
"retention": 1.0025316455696203,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L112",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 112,
"raw_line": "| auto_round | Q4_K_M | 5.12 | 0.4545 | 0.792 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (repeat 5) (AutoRound-generated; actual 5.12 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.4323,
"retention": 0.9203747072599532,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L113",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 113,
"raw_line": "| auto_round | Q4_K_M (repeat 5) | 5.12 | 0.4323 | 0.7958 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. This quant row is a 5-repeat average while the F16 baseline row is a single run. GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF Q4_K_M (repeat 5)",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (repeat 5) (AutoRound-generated; actual 5.12 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.7958,
"retention": 1.0073417721518987,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L113",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 113,
"raw_line": "| auto_round | Q4_K_M (repeat 5) | 5.12 | 0.4323 | 0.7958 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. This quant row is a 5-repeat average while the F16 baseline row is a single run. GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "auto_round Qwen3.5-4B GGUF Q4_K_M (repeat 5)",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (unsloth file as labelled; actual 5.19 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.3838,
"retention": 0.8171173089205875,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L114",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 114,
"raw_line": "| unsloth | Q4_K_M | 5.19 | 0.3838 | 0.794 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (unsloth file as labelled; actual 5.19 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.794,
"retention": 1.0050632911392405,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L114",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 114,
"raw_line": "| unsloth | Q4_K_M | 5.19 | 0.3838 | 0.794 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. Single run per row (the doc's separate '(repeat 5)' rows are recorded separately). GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF Q4_K_M",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (repeat 5) (unsloth file as labelled; actual 5.19 bits/weight)",
"metric": "GPQA Diamond (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.4697,
"quant_score": 0.4424,
"retention": 0.9418777943368107,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L115",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 115,
"raw_line": "| unsloth | Q4_K_M (repeat 5) | 5.19 | 0.4424 | 0.7906 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. This quant row is a 5-repeat average while the F16 baseline row is a single run. GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF Q4_K_M (repeat 5)",
"artifact_revision": null
},
{
"model": "Qwen3.5-4B",
"model_id": "qwen3-5-4b",
"params_b": 4.66,
"quant": "GGUF Q4_K_M (repeat 5) (unsloth file as labelled; actual 5.19 bits/weight)",
"metric": "MATH-500 (EvalScope via llama-server, --reasoning off, --predict 2048, ctx 65536, --parallel 16)",
"full": 0.79,
"quant_score": 0.7906,
"retention": 1.000759493670886,
"baseline_quant": "F16 GGUF (same table, line 104)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/auto_scheme_acc.md#L115",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 115,
"raw_line": "| unsloth | Q4_K_M (repeat 5) | 5.19 | 0.4424 | 0.7906 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/auto_scheme_acc.md",
"sha256": "1413e30d50f3e2325d88d932d09ca108a532ea7f2ecb0be8ea5f1ef9013a19f3",
"line": 104,
"raw_line": "| f16 | f16 | 16 | 0.4697 | 0.79 |"
},
"transformation": "Published fractions used as printed (already 0-1); retention = quant_score / baseline_score recomputed, not copied from any printed percentage",
"note": "Intel auto-round maintainers served each GGUF with llama-server (flags on lines 65-75) and scored with EvalScope. Reasoning disabled and 2048-token cap, so absolute scores are not comparable to thinking-mode results. This quant row is a 5-repeat average while the F16 baseline row is a single run. GPQA Diamond has 198 questions, so small differences are within sampling noise.",
"aggregation": "single benchmark",
"study": "intel/auto-round@a4f01f8:auto_scheme_acc:evalscope:Qwen3.5-4B",
"artifact": "unsloth Qwen3.5-4B GGUF Q4_K_M (repeat 5)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "MXFP4 via AutoRound 'RTN' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6545,
"quant_score": 0.6054,
"retention": 0.9249809014514895,
"baseline_quant": "BF16 (unquantized, same table line 6)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L7",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 7,
"raw_line": "| RTN | 49.23 | 76.30 | 85.63 | 51.93 | 58.45 | 66.18 | 27.40 | 73.12 | 33.29 | 64.48 | 79.98 | 60.54 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 6,
"raw_line": "| **Qwen3-8B**BF16 | 55.46 | 83.38 | 86.73 | 57.04 | 64.00 | 72.96 | 31.20 | 76.61 | 36.72 | 68.27 | 87.57 | 65.45 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' MXFP4 results (docs/opt_rtn.md). MXFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:MXFP4:Qwen3-8B",
"artifact": "Qwen3-8B MXFP4 RTN",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "MXFP4 via AutoRound 'OPT-RTN(-2,-1,1,2)' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6545,
"quant_score": 0.6178,
"retention": 0.9439266615737203,
"baseline_quant": "BF16 (unquantized, same table line 6)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L8",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 8,
"raw_line": "| OPT-RTN(-2,-1,1,2) | 50.17 | 79.25 | 84.98 | 53.18 | 60.20 | 66.88 | 28.80 | 73.83 | 34.88 | 65.43 | 82.03 | 61.78 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 6,
"raw_line": "| **Qwen3-8B**BF16 | 55.46 | 83.38 | 86.73 | 57.04 | 64.00 | 72.96 | 31.20 | 76.61 | 36.72 | 68.27 | 87.57 | 65.45 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' MXFP4 results (docs/opt_rtn.md). MXFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:MXFP4:Qwen3-8B",
"artifact": "Qwen3-8B MXFP4 OPT-RTN(-2,-1,1,2)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "MXFP4 via AutoRound 'OPT-RTN(-1,1) (Our's default)' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6545,
"quant_score": 0.617,
"retention": 0.9427043544690602,
"baseline_quant": "BF16 (unquantized, same table line 6)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L9",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 9,
"raw_line": "| OPT-RTN(-1,1) (Our's default) | 49.83 | 79.04 | 85.54 | 53.02 | 60.00 | 67.26 | 28.60 | 73.67 | 35.01 | 63.77 | 82.94 | 61.70 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 6,
"raw_line": "| **Qwen3-8B**BF16 | 55.46 | 83.38 | 86.73 | 57.04 | 64.00 | 72.96 | 31.20 | 76.61 | 36.72 | 68.27 | 87.57 | 65.45 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' MXFP4 results (docs/opt_rtn.md). MXFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:MXFP4:Qwen3-8B",
"artifact": "Qwen3-8B MXFP4 OPT-RTN(-1,1) (Our's default)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "MXFP4 via AutoRound 'orig-RTN' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6517,
"quant_score": 0.5721,
"retention": 0.8778579100813257,
"baseline_quant": "BF16 (unquantized, same table line 11)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L12",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 12,
"raw_line": "| orig-RTN | 46.08 | 76.64 | 80.37 | 55.60 | 61.42 | 57.11 | 32.00 | 75.52 | 28.15 | 69.69 | 46.70 | 57.21 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 11,
"raw_line": "| **Llama3.1-8B-I** BF16 | 51.54 | 81.78 | 84.04 | 59.13 | 73.14 | 67.96 | 33.00 | 80.03 | 37.21 | 73.80 | 75.28 | 65.17 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' MXFP4 results (docs/opt_rtn.md). MXFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:MXFP4:Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"artifact": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I') MXFP4 orig-RTN",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "MXFP4 via AutoRound 'OPT-RTN(-2,-1,1,2)' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6517,
"quant_score": 0.5848,
"retention": 0.8973454043271443,
"baseline_quant": "BF16 (unquantized, same table line 11)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L13",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 13,
"raw_line": "| OPT-RTN(-2,-1,1,2) | 45.48 | 77.78 | 80.83 | 56.10 | 64.62 | 58.81 | 32.40 | 76.44 | 29.01 | 69.69 | 52.08 | 58.48 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 11,
"raw_line": "| **Llama3.1-8B-I** BF16 | 51.54 | 81.78 | 84.04 | 59.13 | 73.14 | 67.96 | 33.00 | 80.03 | 37.21 | 73.80 | 75.28 | 65.17 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' MXFP4 results (docs/opt_rtn.md). MXFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:MXFP4:Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"artifact": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I') MXFP4 OPT-RTN(-2,-1,1,2)",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "MXFP4 via AutoRound 'OPT-RNT(-1,1) (Our's default)' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6517,
"quant_score": 0.5914,
"retention": 0.9074727635415069,
"baseline_quant": "BF16 (unquantized, same table line 11)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L14",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 14,
"raw_line": "| OPT-RNT(-1,1) (Our's default) | 46.25 | 76.39 | 80.86 | 55.88 | 65.17 | 58.77 | 33.60 | 77.64 | 30.48 | 70.56 | 54.89 | 59.14 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 11,
"raw_line": "| **Llama3.1-8B-I** BF16 | 51.54 | 81.78 | 84.04 | 59.13 | 73.14 | 67.96 | 33.00 | 80.03 | 37.21 | 73.80 | 75.28 | 65.17 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' MXFP4 results (docs/opt_rtn.md). MXFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:MXFP4:Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"artifact": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I') MXFP4 OPT-RNT(-1,1) (Our's default)",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 via AutoRound 'RTN' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6545,
"quant_score": 0.6401,
"retention": 0.977998472116119,
"baseline_quant": "BF16 (unquantized, same table line 20)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L21",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 21,
"raw_line": "| RTN | 54.35 | 81.65 | 86.02 | 55.36 | 62.31 | 70.93 | 30.20 | 75.79 | 34.76 | 68.59 | 84.15 | 64.01 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 20,
"raw_line": "| **Qwen3-8B** | 55.46 | 83.38 | 86.73 | 57.04 | 64.00 | 72.96 | 31.20 | 76.61 | 36.72 | 68.27 | 87.57 | 65.45 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' NVFP4 results (docs/opt_rtn.md). NVFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:NVFP4:Qwen3-8B",
"artifact": "Qwen3-8B NVFP4 RTN",
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 via AutoRound 'opt-rtn' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6545,
"quant_score": 0.644,
"retention": 0.9839572192513368,
"baseline_quant": "BF16 (unquantized, same table line 20)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L22",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 22,
"raw_line": "| opt-rtn | 54.69 | 82.24 | 85.72 | 55.09 | 63.77 | 71.33 | 31.60 | 75.52 | 35.74 | 66.30 | 86.35 | 64.40 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 20,
"raw_line": "| **Qwen3-8B** | 55.46 | 83.38 | 86.73 | 57.04 | 64.00 | 72.96 | 31.20 | 76.61 | 36.72 | 68.27 | 87.57 | 65.45 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' NVFP4 results (docs/opt_rtn.md). NVFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:NVFP4:Qwen3-8B",
"artifact": "Qwen3-8B NVFP4 opt-rtn",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "NVFP4 via AutoRound 'RTN' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6517,
"quant_score": 0.6265,
"retention": 0.9613319011815251,
"baseline_quant": "BF16 (unquantized, same table line 24)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L25",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 25,
"raw_line": "| RTN | 49.15 | 79.25 | 83.24 | 58.29 | 71.20 | 64.05 | 35.00 | 78.62 | 33.41 | 72.85 | 64.06 | 62.65 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 24,
"raw_line": "| **Llama3.1-8B-I** | 51.54 | 81.78 | 84.04 | 59.13 | 73.14 | 67.96 | 33.00 | 80.03 | 37.21 | 73.80 | 75.28 | 65.17 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' NVFP4 results (docs/opt_rtn.md). NVFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:NVFP4:Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"artifact": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I') NVFP4 RTN",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "NVFP4 via AutoRound 'opt-rtn' (iters 0; scheme as defined by auto-round; activation handling not restated in this table)",
"metric": "Average of 11 lm-eval tasks (ARC-c, ARC-e, BoolQ, HellaSwag, LAMBADA, MMLU, OBQA, PIQA, TruthfulQA, WinoGrande, GSM8K strict)",
"full": 0.6517,
"quant_score": 0.6391,
"retention": 0.9806659505907626,
"baseline_quant": "BF16 (unquantized, same table line 24)",
"source": "https://github.com/intel/auto-round/blob/a4f01f8584a69f4af8376239a5fa51331a6747be/docs/opt_rtn.md#L26",
"source_id": "quant-auto-round-docs",
"evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 26,
"raw_line": "| opt-rtn | 51.02 | 80.13 | 84.53 | 58.40 | 71.34 | 64.76 | 35.60 | 79.43 | 35.62 | 72.93 | 69.29 | 63.91 |"
},
"baseline_evidence": {
"file": "speed-evidence/intel__auto-round/docs/opt_rtn.md",
"sha256": "b96115502a998af5d42729a697e8c03a1c237fe2515398fdbd65caf914724d6d",
"line": 24,
"raw_line": "| **Llama3.1-8B-I** | 51.54 | 81.78 | 84.04 | 59.13 | 73.14 | 67.96 | 33.00 | 80.03 | 37.21 | 73.80 | 75.28 | 65.17 |"
},
"transformation": "Published percentages / 100; retention = quant_score / BF16_score recomputed",
"note": "Intel auto-round maintainers' NVFP4 results (docs/opt_rtn.md). NVFP4 is a 4-bit floating-point microscaling format aimed at Blackwell/accelerator kernels, not a GGUF K-quant; whether real kernels or fake quantization were used is not stated here.",
"aggregation": "suite average of 11 tasks",
"study": "intel/auto-round@a4f01f8:opt_rtn:NVFP4:Llama-3.1-8B-Instruct ('Llama3.1-8B-I')",
"artifact": "Llama-3.1-8B-Instruct ('Llama3.1-8B-I') NVFP4 opt-rtn",
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "ONNX INT4 AWQ (NVIDIA ModelOpt-Windows v0.19.0, quantizing the FP16 ONNX model)",
"metric": "MMLU 5-shot (ModelOpt-Windows accuracy_benchmark; onnxruntime-genai-directml 0.4, transformers 4.44)",
"full": 0.6845,
"quant_score": 0.661,
"retention": 0.9656683710737763,
"baseline_quant": "ONNX FP16 (GenAI Model Builder, DirectML EP; same table row)",
"source": "https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/ed7e87953c1a3ed6a21c8bf6eac90565fa6b7a36/examples/windows/Benchmark.md#L39",
"source_id": "quant-tensorrt-model-optimizer-windows",
"evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 39,
"raw_line": "| [Llama3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | 68.45 | 66.1 |"
},
"baseline_evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 39,
"raw_line": "| [Llama3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | 68.45 | 66.1 |"
},
"transformation": "MMLU percentages / 100; retention = INT4 / FP16 recomputed. Baseline and quant values sit in the same table row.",
"note": "NVIDIA ModelOpt-Windows benchmark reference (Windows, RTX 4090, DirectML). ONNX runtime artefacts, not GGUF/MLX. Parameter count for unmapped models is nominal from the model name.",
"aggregation": "single benchmark",
"study": "NVIDIA/TensorRT-Model-Optimizer@ed7e879:examples/windows/Benchmark.md:MMLU",
"artifact": "Llama-3.1-8B-Instruct ONNX INT4 AWQ (ModelOpt-Windows)",
"artifact_revision": null
},
{
"model": "Mistral-7B-Instruct-v0.3",
"model_id": null,
"params_b": 7,
"quant": "ONNX INT4 AWQ (NVIDIA ModelOpt-Windows v0.19.0, quantizing the FP16 ONNX model)",
"metric": "MMLU 5-shot (ModelOpt-Windows accuracy_benchmark; onnxruntime-genai-directml 0.4, transformers 4.44)",
"full": 0.6176,
"quant_score": 0.6073,
"retention": 0.9833225388601037,
"baseline_quant": "ONNX FP16 (GenAI Model Builder, DirectML EP; same table row)",
"source": "https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/ed7e87953c1a3ed6a21c8bf6eac90565fa6b7a36/examples/windows/Benchmark.md#L41",
"source_id": "quant-tensorrt-model-optimizer-windows",
"evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 41,
"raw_line": "| [Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | 61.76 | 60.73 |"
},
"baseline_evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 41,
"raw_line": "| [Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) | 61.76 | 60.73 |"
},
"transformation": "MMLU percentages / 100; retention = INT4 / FP16 recomputed. Baseline and quant values sit in the same table row.",
"note": "NVIDIA ModelOpt-Windows benchmark reference (Windows, RTX 4090, DirectML). ONNX runtime artefacts, not GGUF/MLX. Parameter count for unmapped models is nominal from the model name.",
"aggregation": "single benchmark",
"study": "NVIDIA/TensorRT-Model-Optimizer@ed7e879:examples/windows/Benchmark.md:MMLU",
"artifact": "Mistral-7B-Instruct-v0.3 ONNX INT4 AWQ (ModelOpt-Windows)",
"artifact_revision": null
},
{
"model": "Llama-3.2-3B-Instruct",
"model_id": "llama-3-2-3b",
"params_b": 3.21,
"quant": "ONNX INT4 AWQ (NVIDIA ModelOpt-Windows v0.19.0, quantizing the FP16 ONNX model)",
"metric": "MMLU 5-shot (ModelOpt-Windows accuracy_benchmark; onnxruntime-genai-directml 0.4, transformers 4.44)",
"full": 0.608,
"quant_score": 0.5771,
"retention": 0.9491776315789475,
"baseline_quant": "ONNX FP16 (GenAI Model Builder, DirectML EP; same table row)",
"source": "https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/ed7e87953c1a3ed6a21c8bf6eac90565fa6b7a36/examples/windows/Benchmark.md#L42",
"source_id": "quant-tensorrt-model-optimizer-windows",
"evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 42,
"raw_line": "| [Llama3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | 60.8 | 57.71 |"
},
"baseline_evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 42,
"raw_line": "| [Llama3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | 60.8 | 57.71 |"
},
"transformation": "MMLU percentages / 100; retention = INT4 / FP16 recomputed. Baseline and quant values sit in the same table row.",
"note": "NVIDIA ModelOpt-Windows benchmark reference (Windows, RTX 4090, DirectML). ONNX runtime artefacts, not GGUF/MLX. Parameter count for unmapped models is nominal from the model name.",
"aggregation": "single benchmark",
"study": "NVIDIA/TensorRT-Model-Optimizer@ed7e879:examples/windows/Benchmark.md:MMLU",
"artifact": "Llama-3.2-3B-Instruct ONNX INT4 AWQ (ModelOpt-Windows)",
"artifact_revision": null
},
{
"model": "Gemma-2b-it (Gemma 1)",
"model_id": null,
"params_b": 2,
"quant": "ONNX INT4 AWQ (NVIDIA ModelOpt-Windows v0.19.0, quantizing the FP16 ONNX model)",
"metric": "MMLU 5-shot (ModelOpt-Windows accuracy_benchmark; onnxruntime-genai-directml 0.4, transformers 4.44)",
"full": 0.3701,
"quant_score": 0.372,
"retention": 1.0051337476357742,
"baseline_quant": "ONNX FP16 (GenAI Model Builder, DirectML EP; same table row)",
"source": "https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/ed7e87953c1a3ed6a21c8bf6eac90565fa6b7a36/examples/windows/Benchmark.md#L43",
"source_id": "quant-tensorrt-model-optimizer-windows",
"evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 43,
"raw_line": "| [Gemma-2b-it](https://huggingface.co/google/gemma-2b-it) | 37.01 | 37.2 |"
},
"baseline_evidence": {
"file": "speed-evidence/NVIDIA__TensorRT-Model-Optimizer/examples/windows/Benchmark.md",
"sha256": "33fd6799f1c25b08d6d87181e08726ae9d032719e391f432f78ae4f6308df5a3",
"line": 43,
"raw_line": "| [Gemma-2b-it](https://huggingface.co/google/gemma-2b-it) | 37.01 | 37.2 |"
},
"transformation": "MMLU percentages / 100; retention = INT4 / FP16 recomputed. Baseline and quant values sit in the same table row.",
"note": "NVIDIA ModelOpt-Windows benchmark reference (Windows, RTX 4090, DirectML). ONNX runtime artefacts, not GGUF/MLX. Parameter count for unmapped models is nominal from the model name.",
"aggregation": "single benchmark",
"study": "NVIDIA/TensorRT-Model-Optimizer@ed7e879:examples/windows/Benchmark.md:MMLU",
"artifact": "Gemma-2b-it (Gemma 1) ONNX INT4 AWQ (ModelOpt-Windows)",
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.5543333333,
"quant_score": 0.5231666667,
"retention": 0.9437763079942139,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.3",
"raw_cells": [
"AWQ",
"78.5",
"72.5",
"60.1",
"44.7",
"32.3",
"25.8"
],
"cells_used": [
"78.5",
"72.5",
"60.1",
"44.7",
"32.3",
"25.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
],
"cells_used": [
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.5543333333,
"quant_score": 0.5291666667,
"retention": 0.9546001203821166,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.4",
"raw_cells": [
"BnB",
"79.6",
"73.3",
"61.5",
"45.9",
"32.3",
"24.9"
],
"cells_used": [
"79.6",
"73.3",
"61.5",
"45.9",
"32.3",
"24.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
],
"cells_used": [
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.5543333333,
"quant_score": 0.5475,
"retention": 0.9876728803961318,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.5",
"raw_cells": [
"GGUF",
"79.7",
"74.3",
"64.2",
"48.5",
"35.8",
"26.0"
],
"cells_used": [
"79.7",
"74.3",
"64.2",
"48.5",
"35.8",
"26.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
],
"cells_used": [
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.5543333333,
"quant_score": 0.5308333333,
"retention": 0.9576067348140473,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.6",
"raw_cells": [
"AutoRound",
"78.2",
"73.3",
"62.2",
"45.8",
"31.3",
"27.7"
],
"cells_used": [
"78.2",
"73.3",
"62.2",
"45.8",
"31.3",
"27.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
],
"cells_used": [
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7018333333,
"quant_score": 0.6901666667,
"retention": 0.9833768701963134,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.8",
"raw_cells": [
"AWQ",
"85.0",
"80.1",
"77.6",
"74.1",
"55.4",
"41.9"
],
"cells_used": [
"85.0",
"80.1",
"77.6",
"74.1",
"55.4",
"41.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
],
"cells_used": [
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7018333333,
"quant_score": 0.6918333333,
"retention": 0.985751602943992,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.9",
"raw_cells": [
"BnB",
"85.4",
"80.0",
"77.6",
"74.0",
"55.1",
"43.0"
],
"cells_used": [
"85.4",
"80.0",
"77.6",
"74.0",
"55.1",
"43.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
],
"cells_used": [
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7018333333,
"quant_score": 0.6943333333,
"retention": 0.9893137022079941,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.10",
"raw_cells": [
"GGUF",
"85.6",
"80.6",
"78.2",
"75.2",
"56.4",
"40.6"
],
"cells_used": [
"85.6",
"80.6",
"78.2",
"75.2",
"56.4",
"40.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
],
"cells_used": [
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7018333333,
"quant_score": 0.6886666667,
"retention": 0.9812396106379121,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.11",
"raw_cells": [
"AutoRound",
"85.4",
"80.3",
"77.9",
"74.4",
"55.2",
"40.0"
],
"cells_used": [
"85.4",
"80.3",
"77.9",
"74.4",
"55.2",
"40.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
],
"cells_used": [
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.6938333333,
"quant_score": 0.6765,
"retention": 0.9750180159007936,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.13",
"raw_cells": [
"AWQ",
"80.5",
"77.5",
"76.6",
"73.0",
"55.7",
"42.6"
],
"cells_used": [
"80.5",
"77.5",
"76.6",
"73.0",
"55.7",
"42.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
],
"cells_used": [
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.6938333333,
"quant_score": 0.6806666667,
"retention": 0.9810233005996168,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.14",
"raw_cells": [
"BnB",
"80.8",
"77.3",
"76.0",
"73.3",
"55.9",
"45.1"
],
"cells_used": [
"80.8",
"77.3",
"76.0",
"73.3",
"55.9",
"45.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
],
"cells_used": [
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.6938333333,
"quant_score": 0.6828333333,
"retention": 0.9841460485219383,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.15",
"raw_cells": [
"GGUF",
"81.2",
"77.2",
"77.7",
"74.6",
"55.0",
"44.0"
],
"cells_used": [
"81.2",
"77.2",
"77.7",
"74.6",
"55.0",
"44.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
],
"cells_used": [
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.6938333333,
"quant_score": 0.6805,
"retention": 0.9807830891655434,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.16",
"raw_cells": [
"AutoRound",
"80.3",
"77.4",
"77.0",
"74.1",
"54.6",
"44.9"
],
"cells_used": [
"80.3",
"77.4",
"77.0",
"74.1",
"54.6",
"44.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
],
"cells_used": [
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7515,
"quant_score": 0.7471666667,
"retention": 0.9942337547571524,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.18",
"raw_cells": [
"AWQ",
"86.5",
"81.6",
"81.5",
"79.6",
"64.7",
"54.4"
],
"cells_used": [
"86.5",
"81.6",
"81.5",
"79.6",
"64.7",
"54.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
],
"cells_used": [
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7515,
"quant_score": 0.7428333333,
"retention": 0.9884675093812376,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.19",
"raw_cells": [
"BnB",
"86.8",
"81.5",
"81.4",
"79.6",
"64.5",
"51.9"
],
"cells_used": [
"86.8",
"81.5",
"81.4",
"79.6",
"64.5",
"51.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
],
"cells_used": [
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7515,
"quant_score": 0.7498333333,
"retention": 0.99778221330672,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.20",
"raw_cells": [
"GGUF",
"87.2",
"81.7",
"81.7",
"79.8",
"65.4",
"54.1"
],
"cells_used": [
"87.2",
"81.7",
"81.7",
"79.8",
"65.4",
"54.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
],
"cells_used": [
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7515,
"quant_score": 0.746,
"retention": 0.9926813040585496,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.21",
"raw_cells": [
"AutoRound",
"86.8",
"81.7",
"81.8",
"79.6",
"64.9",
"52.8"
],
"cells_used": [
"86.8",
"81.7",
"81.8",
"79.6",
"64.9",
"52.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
],
"cells_used": [
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7758333333,
"quant_score": 0.7716666667,
"retention": 0.9946294308053545,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.23",
"raw_cells": [
"AWQ",
"85.2",
"80.8",
"82.7",
"80.5",
"67.0",
"66.8"
],
"cells_used": [
"85.2",
"80.8",
"82.7",
"80.5",
"67.0",
"66.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
],
"cells_used": [
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7758333333,
"quant_score": 0.7618333333,
"retention": 0.9819548872172698,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.24",
"raw_cells": [
"BnB",
"84.8",
"80.5",
"81.9",
"79.3",
"65.1",
"65.5"
],
"cells_used": [
"84.8",
"80.5",
"81.9",
"79.3",
"65.1",
"65.5"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
],
"cells_used": [
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7758333333,
"quant_score": 0.7725,
"retention": 0.9957035446185049,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.25",
"raw_cells": [
"GGUF",
"85.4",
"80.9",
"82.4",
"80.5",
"67.2",
"67.1"
],
"cells_used": [
"85.4",
"80.9",
"82.4",
"80.5",
"67.2",
"67.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
],
"cells_used": [
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7758333333,
"quant_score": 0.773,
"retention": 0.9963480129321739,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T1",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.26",
"raw_cells": [
"AutoRound",
"85.5",
"80.8",
"82.7",
"80.5",
"67.4",
"66.9"
],
"cells_used": [
"85.5",
"80.8",
"82.7",
"80.5",
"67.4",
"66.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T1.2",
"html_row_id": "S4.T1.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
],
"cells_used": [
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T1.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7286666667,
"quant_score": 0.7148333333,
"retention": 0.9810155534317925,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.3",
"raw_cells": [
"AWQ",
"78.4",
"80.9",
"76.9",
"75.7",
"73.0",
"44.0"
],
"cells_used": [
"78.4",
"80.9",
"76.9",
"75.7",
"73.0",
"44.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
],
"cells_used": [
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7286666667,
"quant_score": 0.7205,
"retention": 0.9887923146848677,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.4",
"raw_cells": [
"BnB",
"78.7",
"81.3",
"77.3",
"76.9",
"74.4",
"43.7"
],
"cells_used": [
"78.7",
"81.3",
"77.3",
"76.9",
"74.4",
"43.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
],
"cells_used": [
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7286666667,
"quant_score": 0.7251666667,
"retention": 0.99519670631312,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.5",
"raw_cells": [
"GGUF",
"79.2",
"81.4",
"77.6",
"78.0",
"75.4",
"43.5"
],
"cells_used": [
"79.2",
"81.4",
"77.6",
"78.0",
"75.4",
"43.5"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
],
"cells_used": [
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7286666667,
"quant_score": 0.718,
"retention": 0.9853613906228105,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.6",
"raw_cells": [
"AutoRound",
"79.0",
"81.0",
"77.0",
"76.8",
"74.1",
"42.9"
],
"cells_used": [
"79.0",
"81.0",
"77.0",
"76.8",
"74.1",
"42.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
],
"cells_used": [
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7816666667,
"quant_score": 0.774,
"retention": 0.9901918976123586,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.8",
"raw_cells": [
"AWQ",
"82.1",
"83.6",
"81.1",
"83.0",
"81.9",
"52.7"
],
"cells_used": [
"82.1",
"83.6",
"81.1",
"83.0",
"81.9",
"52.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
],
"cells_used": [
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7816666667,
"quant_score": 0.7763333333,
"retention": 0.9931769721964531,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.9",
"raw_cells": [
"BnB",
"82.3",
"83.7",
"81.2",
"83.0",
"81.6",
"54.0"
],
"cells_used": [
"82.3",
"83.7",
"81.2",
"83.0",
"81.6",
"54.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
],
"cells_used": [
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7816666667,
"quant_score": 0.7781666667,
"retention": 0.9955223880598925,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.10",
"raw_cells": [
"GGUF",
"82.5",
"84.0",
"81.2",
"83.4",
"82.4",
"53.4"
],
"cells_used": [
"82.5",
"84.0",
"81.2",
"83.4",
"82.4",
"53.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
],
"cells_used": [
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7816666667,
"quant_score": 0.7735,
"retention": 0.9895522387637717,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.11",
"raw_cells": [
"AutoRound",
"82.3",
"83.6",
"81.2",
"82.9",
"81.6",
"52.5"
],
"cells_used": [
"82.3",
"83.6",
"81.2",
"82.9",
"81.6",
"52.5"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.7",
"raw_cells": [
"Qwen3-8B",
"-",
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
],
"cells_used": [
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7373333333,
"quant_score": 0.7086666667,
"retention": 0.9611211574123473,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.13",
"raw_cells": [
"AWQ",
"78.2",
"81.0",
"78.3",
"68.2",
"62.5",
"57.0"
],
"cells_used": [
"78.2",
"81.0",
"78.3",
"68.2",
"62.5",
"57.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
],
"cells_used": [
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7373333333,
"quant_score": 0.6998333333,
"retention": 0.9491410488222939,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.14",
"raw_cells": [
"BnB",
"77.8",
"81.3",
"77.9",
"64.5",
"62.1",
"56.3"
],
"cells_used": [
"77.8",
"81.3",
"77.9",
"64.5",
"62.1",
"56.3"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
],
"cells_used": [
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7373333333,
"quant_score": 0.7228333333,
"retention": 0.9803345388779537,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.15",
"raw_cells": [
"GGUF",
"78.5",
"81.7",
"78.9",
"68.0",
"68.8",
"57.8"
],
"cells_used": [
"78.5",
"81.7",
"78.9",
"68.0",
"68.8",
"57.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
],
"cells_used": [
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7373333333,
"quant_score": 0.7041666667,
"retention": 0.9550180832710227,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.16",
"raw_cells": [
"AutoRound",
"77.7",
"81.2",
"78.5",
"61.0",
"67.1",
"57.0"
],
"cells_used": [
"77.7",
"81.2",
"78.5",
"61.0",
"67.1",
"57.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.12",
"raw_cells": [
"Llama-3.1-8B",
"-",
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
],
"cells_used": [
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.8013333333,
"quant_score": 0.8008333333,
"retention": 0.9993760399334183,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.18",
"raw_cells": [
"AWQ",
"82.7",
"84.5",
"82.5",
"84.4",
"84.0",
"62.4"
],
"cells_used": [
"82.7",
"84.5",
"82.5",
"84.4",
"84.0",
"62.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
],
"cells_used": [
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.8013333333,
"quant_score": 0.7993333333,
"retention": 0.9975041597336732,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.19",
"raw_cells": [
"BnB",
"82.5",
"84.1",
"82.2",
"84.5",
"83.9",
"62.4"
],
"cells_used": [
"82.5",
"84.1",
"82.2",
"84.5",
"83.9",
"62.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
],
"cells_used": [
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.8013333333,
"quant_score": 0.7998333333,
"retention": 0.9981281198002548,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.20",
"raw_cells": [
"GGUF",
"82.8",
"84.4",
"82.5",
"84.1",
"83.9",
"62.2"
],
"cells_used": [
"82.8",
"84.4",
"82.5",
"84.1",
"83.9",
"62.2"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
],
"cells_used": [
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.8013333333,
"quant_score": 0.7998333333,
"retention": 0.9981281198002548,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.21",
"raw_cells": [
"AutoRound",
"82.8",
"84.5",
"82.4",
"84.3",
"84.0",
"61.9"
],
"cells_used": [
"82.8",
"84.5",
"82.4",
"84.3",
"84.0",
"61.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.17",
"raw_cells": [
"Qwen3-32B",
"-",
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
],
"cells_used": [
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "AWQ 4-bit (AutoAWQ, zero point, group size 128)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.795,
"quant_score": 0.7948333333,
"retention": 0.9997903563522011,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.23",
"raw_cells": [
"AWQ",
"80.8",
"83.3",
"81.2",
"82.7",
"82.2",
"66.7"
],
"cells_used": [
"80.8",
"83.3",
"81.2",
"82.7",
"82.2",
"66.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
],
"cells_used": [
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "bitsandbytes NF4 (Transformers integration, nested/double quantization)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.795,
"quant_score": 0.7683333333,
"retention": 0.9664570230188678,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.24",
"raw_cells": [
"BnB",
"80.0",
"82.5",
"80.2",
"78.2",
"79.4",
"60.7"
],
"cells_used": [
"80.0",
"82.5",
"80.2",
"78.2",
"79.4",
"60.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
],
"cells_used": [
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "GGUF Q4_K_M with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.795,
"quant_score": 0.793,
"retention": 0.9974842767295597,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.25",
"raw_cells": [
"GGUF",
"80.8",
"83.2",
"80.9",
"82.6",
"82.0",
"66.3"
],
"cells_used": [
"80.8",
"83.2",
"80.9",
"82.6",
"82.0",
"66.3"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
],
"cells_used": [
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "AutoRound 4-bit (group size 128; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.795,
"quant_score": 0.7946666667,
"retention": 0.9995807128301886,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T2",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.26",
"raw_cells": [
"AutoRound",
"81.2",
"83.4",
"81.2",
"82.7",
"82.0",
"66.3"
],
"cells_used": [
"81.2",
"83.4",
"81.2",
"82.7",
"82.0",
"66.3"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T2.2",
"html_row_id": "S4.T2.2.22",
"raw_cells": [
"Llama-3.3-70B",
"-",
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
],
"cells_used": [
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T2.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.5543333333,
"quant_score": 0.4491666667,
"retention": 0.8102826218767458,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.3",
"raw_cells": [
"GGUF",
"69.4",
"66.2",
"48.2",
"30.8",
"23.7",
"31.2"
],
"cells_used": [
"69.4",
"66.2",
"48.2",
"30.8",
"23.7",
"31.2"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
],
"cells_used": [
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.5543333333,
"quant_score": 0.3016666667,
"retention": 0.5441972340074682,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.4",
"raw_cells": [
"AutoRound",
"33.1",
"34.8",
"32.9",
"28.6",
"24.5",
"27.1"
],
"cells_used": [
"33.1",
"34.8",
"32.9",
"28.6",
"24.5",
"27.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
],
"cells_used": [
"80.4",
"74.2",
"64.7",
"50.1",
"35.9",
"27.3"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7018333333,
"quant_score": 0.6226666667,
"retention": 0.8872001900682595,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.6",
"raw_cells": [
"GGUF",
"83.7",
"79.2",
"72.5",
"59.9",
"40.6",
"37.7"
],
"cells_used": [
"83.7",
"79.2",
"72.5",
"59.9",
"40.6",
"37.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.5",
"raw_cells": [
"Qwen3-8B",
"-",
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
],
"cells_used": [
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7018333333,
"quant_score": 0.4321666667,
"retention": 0.615768226151307,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.7",
"raw_cells": [
"AutoRound",
"47.0",
"63.9",
"49.7",
"34.4",
"31.9",
"32.4"
],
"cells_used": [
"47.0",
"63.9",
"49.7",
"34.4",
"31.9",
"32.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.5",
"raw_cells": [
"Qwen3-8B",
"-",
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
],
"cells_used": [
"85.7",
"80.8",
"78.6",
"76.1",
"57.9",
"42.0"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.6938333333,
"quant_score": 0.5565,
"retention": 0.8020658179583025,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.9",
"raw_cells": [
"GGUF",
"72.7",
"73.4",
"66.8",
"46.2",
"39.3",
"35.5"
],
"cells_used": [
"72.7",
"73.4",
"66.8",
"46.2",
"39.3",
"35.5"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.8",
"raw_cells": [
"Llama-3.1-8B",
"-",
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
],
"cells_used": [
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.6938333333,
"quant_score": 0.3573333333,
"retention": 0.5150132116029312,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.10",
"raw_cells": [
"AutoRound",
"39.6",
"45.0",
"39.6",
"32.9",
"27.7",
"29.6"
],
"cells_used": [
"39.6",
"45.0",
"39.6",
"32.9",
"27.7",
"29.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.8",
"raw_cells": [
"Llama-3.1-8B",
"-",
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
],
"cells_used": [
"81.6",
"77.7",
"78.0",
"75.6",
"57.9",
"45.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7515,
"quant_score": 0.699,
"retention": 0.9301397205588823,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.12",
"raw_cells": [
"GGUF",
"86.0",
"80.5",
"78.1",
"74.7",
"56.1",
"44.0"
],
"cells_used": [
"86.0",
"80.5",
"78.1",
"74.7",
"56.1",
"44.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.11",
"raw_cells": [
"Qwen3-32B",
"-",
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
],
"cells_used": [
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7515,
"quant_score": 0.5915,
"retention": 0.7870924817032602,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.13",
"raw_cells": [
"AutoRound",
"81.1",
"75.0",
"68.0",
"54.5",
"37.3",
"39.0"
],
"cells_used": [
"81.1",
"75.0",
"68.0",
"54.5",
"37.3",
"39.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.11",
"raw_cells": [
"Qwen3-32B",
"-",
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
],
"cells_used": [
"87.1",
"81.7",
"81.8",
"80.0",
"65.4",
"54.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7758333333,
"quant_score": 0.743,
"retention": 0.9576799141120378,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.15",
"raw_cells": [
"GGUF",
"84.2",
"79.8",
"81.0",
"78.9",
"63.2",
"58.7"
],
"cells_used": [
"84.2",
"79.8",
"81.0",
"78.9",
"63.2",
"58.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.14",
"raw_cells": [
"Llama-3.3-70B",
"-",
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
],
"cells_used": [
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ English->X COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7758333333,
"quant_score": 0.6668333333,
"retention": 0.8595059076201721,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T3",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.16",
"raw_cells": [
"AutoRound",
"81.7",
"78.7",
"75.5",
"68.4",
"54.0",
"41.8"
],
"cells_used": [
"81.7",
"78.7",
"75.5",
"68.4",
"54.0",
"41.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T3.2",
"html_row_id": "S4.T3.2.14",
"raw_cells": [
"Llama-3.3-70B",
"-",
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
],
"cells_used": [
"85.6",
"81.0",
"82.9",
"80.9",
"67.6",
"67.5"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T3.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7286666667,
"quant_score": 0.63,
"retention": 0.8645928636383992,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.3",
"raw_cells": [
"GGUF",
"71.9",
"77.8",
"70.9",
"59.6",
"56.2",
"41.6"
],
"cells_used": [
"71.9",
"77.8",
"70.9",
"59.6",
"56.2",
"41.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
],
"cells_used": [
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-1.7B",
"model_id": "qwen3-1-7b",
"params_b": 2.03,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7286666667,
"quant_score": 0.4006666667,
"retention": 0.5498627630581032,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.4",
"raw_cells": [
"AutoRound",
"43.0",
"43.6",
"37.3",
"42.1",
"38.9",
"35.5"
],
"cells_used": [
"43.0",
"43.6",
"37.3",
"42.1",
"38.9",
"35.5"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.2",
"raw_cells": [
"Qwen3-1.7B",
"-",
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
],
"cells_used": [
"79.3",
"81.6",
"77.9",
"78.4",
"76.2",
"43.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Qwen3-1.7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7816666667,
"quant_score": 0.7475,
"retention": 0.9562899786372585,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.6",
"raw_cells": [
"GGUF",
"80.9",
"82.8",
"79.7",
"81.3",
"79.8",
"44.0"
],
"cells_used": [
"80.9",
"82.8",
"79.7",
"81.3",
"79.8",
"44.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.5",
"raw_cells": [
"Qwen3-8B",
"-",
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
],
"cells_used": [
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7816666667,
"quant_score": 0.6285,
"retention": 0.8040511726736012,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.7",
"raw_cells": [
"AutoRound",
"71.4",
"74.3",
"70.1",
"64.6",
"58.7",
"38.0"
],
"cells_used": [
"71.4",
"74.3",
"70.1",
"64.6",
"58.7",
"38.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.5",
"raw_cells": [
"Qwen3-8B",
"-",
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
],
"cells_used": [
"82.6",
"84.0",
"81.5",
"83.6",
"82.5",
"54.8"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7373333333,
"quant_score": 0.6968333333,
"retention": 0.9450723327280774,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.9",
"raw_cells": [
"GGUF",
"76.3",
"79.9",
"76.0",
"70.2",
"68.1",
"47.6"
],
"cells_used": [
"76.3",
"79.9",
"76.0",
"70.2",
"68.1",
"47.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.8",
"raw_cells": [
"Llama-3.1-8B",
"-",
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
],
"cells_used": [
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.1-8B-Instruct",
"model_id": "llama-3-1-8b",
"params_b": 8.03,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.7373333333,
"quant_score": 0.4661666667,
"retention": 0.6322332731298479,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.10",
"raw_cells": [
"AutoRound",
"55.8",
"54.3",
"50.8",
"37.5",
"43.0",
"38.3"
],
"cells_used": [
"55.8",
"54.3",
"50.8",
"37.5",
"43.0",
"38.3"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.8",
"raw_cells": [
"Llama-3.1-8B",
"-",
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
],
"cells_used": [
"79.3",
"81.6",
"79.1",
"72.2",
"71.8",
"58.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Llama-3.1-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.8013333333,
"quant_score": 0.7656666667,
"retention": 0.9554908486670338,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.12",
"raw_cells": [
"GGUF",
"81.0",
"83.1",
"78.8",
"79.9",
"79.5",
"57.1"
],
"cells_used": [
"81.0",
"83.1",
"78.8",
"79.9",
"79.5",
"57.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.11",
"raw_cells": [
"Qwen3-32B",
"-",
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
],
"cells_used": [
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.8013333333,
"quant_score": 0.7585,
"retention": 0.946547421004432,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.13",
"raw_cells": [
"AutoRound",
"80.7",
"82.1",
"79.7",
"81.7",
"79.8",
"51.1"
],
"cells_used": [
"80.7",
"82.1",
"79.7",
"81.7",
"79.8",
"51.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.11",
"raw_cells": [
"Qwen3-32B",
"-",
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
],
"cells_used": [
"82.8",
"84.4",
"82.4",
"84.6",
"84.2",
"62.4"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "GGUF Q2_K with imatrix (llama.cpp; imatrix from 20k WikiText samples)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.795,
"quant_score": 0.7758333333,
"retention": 0.9758909852830189,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.15",
"raw_cells": [
"GGUF",
"80.1",
"82.1",
"80.3",
"80.5",
"80.7",
"61.8"
],
"cells_used": [
"80.1",
"82.1",
"80.3",
"80.5",
"80.7",
"61.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.14",
"raw_cells": [
"Llama-3.3-70B",
"-",
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
],
"cells_used": [
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Llama-3.3-70B-Instruct",
"model_id": "llama-3-3-70b",
"params_b": 70,
"quant": "AutoRound 2-bit (group size 32; 512 calibration samples, seqlen 4096, 512 iterations)",
"metric": "WMT24++ X->English COMET (wmt22-comet-da), unweighted mean over 6 language pairs (ja_JP, fr_FR, pl_PL, bn_IN, ml_IN, zu_ZA); greedy decoding",
"full": 0.795,
"quant_score": 0.7671666667,
"retention": 0.9649895178616352,
"baseline_quant": "unquantized original checkpoint (same study; dtype not stated)",
"source": "https://arxiv.org/html/2508.20893v1#S4.T4",
"source_id": "quant-mt-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.16",
"raw_cells": [
"AutoRound",
"79.9",
"81.5",
"79.4",
"79.6",
"80.1",
"59.8"
],
"cells_used": [
"79.9",
"81.5",
"79.4",
"79.6",
"80.1",
"59.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2508.20893v1.html",
"sha256": "75ae7e1d86f14407e64268f0edb05bba1f50527be992489b631db0f24c490f1c",
"html_table_id": "S4.T4.2",
"html_row_id": "S4.T4.2.14",
"raw_cells": [
"Llama-3.3-70B",
"-",
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
],
"cells_used": [
"80.9",
"83.3",
"81.1",
"82.6",
"82.2",
"66.9"
]
},
"transformation": "Six published COMET cells per row averaged by whatmodel.sh (unweighted), then /100; retention = mean_quant / mean_original; not a published average",
"note": "Machine-translation quality only; low-resource pairs (bn, ml, zu) dominate the drop. Qwen3 thinking disabled. Llama 3.3 70B is the Instruct release. COMET is not linearly interpretable; within-study relative evidence only.",
"aggregation": "mean of 6 language-pair COMET scores (computed)",
"study": "arXiv:2508.20893v1:S4.T4.2:Llama-3.3-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-Coder-7B-Instruct",
"model_id": "qwen2-5-coder-7b",
"params_b": 7.62,
"quant": "AWQ 4-bit",
"metric": "BigCodeBench (Python) pass@1",
"full": 0.41,
"quant_score": 0.41,
"retention": 1,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2607.14181v1#S5.T4",
"source_id": "quant-codegen-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.11",
"raw_cells": [
"4 bit",
"AWQ",
"0.41",
"17184\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"0•",
"12\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"631\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"2905\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"2409\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}"
],
"cells_used": [
"0.41"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.10",
"raw_cells": [
"Qwen2.5-Coder-7B",
"16 bit",
"FP",
"0.41",
"17281",
"0",
"21",
"670",
"2975",
"2492"
],
"cells_used": [
"0.41"
]
},
"transformation": "Published pass@1 fractions (two decimals) used as-is; retention = quant / FP16",
"note": "Pass@1 is published to two decimals only, so retention is coarse (+/-0.025 absolute). Pre-quantized Hugging Face checkpoints were used where available, otherwise quantized locally with official tools; the exact checkpoint per method is not identified.",
"aggregation": "single benchmark",
"study": "arXiv:2607.14181v1:BCB-Python:Qwen2.5-Coder-7B-Instruct",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-Coder-7B-Instruct",
"model_id": "qwen2-5-coder-7b",
"params_b": 7.62,
"quant": "GPTQ 4-bit",
"metric": "BigCodeBench (Python) pass@1",
"full": 0.41,
"quant_score": 0.41,
"retention": 1,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2607.14181v1#S5.T4",
"source_id": "quant-codegen-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.12",
"raw_cells": [
"GPTQ",
"0.41",
"17320\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"0•",
"11\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"679\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"2927\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"2458\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}"
],
"cells_used": [
"0.41"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.10",
"raw_cells": [
"Qwen2.5-Coder-7B",
"16 bit",
"FP",
"0.41",
"17281",
"0",
"21",
"670",
"2975",
"2492"
],
"cells_used": [
"0.41"
]
},
"transformation": "Published pass@1 fractions (two decimals) used as-is; retention = quant / FP16",
"note": "Pass@1 is published to two decimals only, so retention is coarse (+/-0.025 absolute). Pre-quantized Hugging Face checkpoints were used where available, otherwise quantized locally with official tools; the exact checkpoint per method is not identified.",
"aggregation": "single benchmark",
"study": "arXiv:2607.14181v1:BCB-Python:Qwen2.5-Coder-7B-Instruct",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-Coder-7B-Instruct",
"model_id": "qwen2-5-coder-7b",
"params_b": 7.62,
"quant": "GGUF Q4_K_M",
"metric": "BigCodeBench (Python) pass@1",
"full": 0.41,
"quant_score": 0.4,
"retention": 0.9756097560975611,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2607.14181v1#S5.T4",
"source_id": "quant-codegen-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.13",
"raw_cells": [
"GGUF",
"0.40",
"17148\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"0•",
"22\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"664\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"2983\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"2565\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}"
],
"cells_used": [
"0.40"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.10",
"raw_cells": [
"Qwen2.5-Coder-7B",
"16 bit",
"FP",
"0.41",
"17281",
"0",
"21",
"670",
"2975",
"2492"
],
"cells_used": [
"0.41"
]
},
"transformation": "Published pass@1 fractions (two decimals) used as-is; retention = quant / FP16",
"note": "Pass@1 is published to two decimals only, so retention is coarse (+/-0.025 absolute). Pre-quantized Hugging Face checkpoints were used where available, otherwise quantized locally with official tools; the exact checkpoint per method is not identified.",
"aggregation": "single benchmark",
"study": "arXiv:2607.14181v1:BCB-Python:Qwen2.5-Coder-7B-Instruct",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-Coder-7B-Instruct",
"model_id": "qwen2-5-coder-7b",
"params_b": 7.62,
"quant": "bitsandbytes NF4 (double quantization)",
"metric": "BigCodeBench (Python) pass@1",
"full": 0.41,
"quant_score": 0.39,
"retention": 0.951219512195122,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2607.14181v1#S5.T4",
"source_id": "quant-codegen-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.14",
"raw_cells": [
"BitsAndBytes",
"0.39",
"17909\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"0•",
"30\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"771\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"3088\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"2718\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}"
],
"cells_used": [
"0.39"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.10",
"raw_cells": [
"Qwen2.5-Coder-7B",
"16 bit",
"FP",
"0.41",
"17281",
"0",
"21",
"670",
"2975",
"2492"
],
"cells_used": [
"0.41"
]
},
"transformation": "Published pass@1 fractions (two decimals) used as-is; retention = quant / FP16",
"note": "Pass@1 is published to two decimals only, so retention is coarse (+/-0.025 absolute). Pre-quantized Hugging Face checkpoints were used where available, otherwise quantized locally with official tools; the exact checkpoint per method is not identified.",
"aggregation": "single benchmark",
"study": "arXiv:2607.14181v1:BCB-Python:Qwen2.5-Coder-7B-Instruct",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-Coder-7B-Instruct",
"model_id": "qwen2-5-coder-7b",
"params_b": 7.62,
"quant": "AQLM 4-bit",
"metric": "BigCodeBench (Python) pass@1",
"full": 0.41,
"quant_score": 0.41,
"retention": 1,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2607.14181v1#S5.T4",
"source_id": "quant-codegen-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.15",
"raw_cells": [
"AQLM",
"0.41",
"17278\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"0•",
"19\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"667\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"2987\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"2577\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}"
],
"cells_used": [
"0.41"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.10",
"raw_cells": [
"Qwen2.5-Coder-7B",
"16 bit",
"FP",
"0.41",
"17281",
"0",
"21",
"670",
"2975",
"2492"
],
"cells_used": [
"0.41"
]
},
"transformation": "Published pass@1 fractions (two decimals) used as-is; retention = quant / FP16",
"note": "Pass@1 is published to two decimals only, so retention is coarse (+/-0.025 absolute). Pre-quantized Hugging Face checkpoints were used where available, otherwise quantized locally with official tools; the exact checkpoint per method is not identified.",
"aggregation": "single benchmark",
"study": "arXiv:2607.14181v1:BCB-Python:Qwen2.5-Coder-7B-Instruct",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-Coder-7B-Instruct",
"model_id": "qwen2-5-coder-7b",
"params_b": 7.62,
"quant": "QuIP# 4-bit",
"metric": "BigCodeBench (Python) pass@1",
"full": 0.41,
"quant_score": 0.32,
"retention": 0.7804878048780488,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2607.14181v1#S5.T4",
"source_id": "quant-codegen-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.16",
"raw_cells": [
"QuIP#",
"0.32",
"16547\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"0•",
"14\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"733\\blacktriangledown{}^{\\color[rgb]{1,0,0}\\blacktriangledown}",
"2841\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}",
"2207\\blacktriangle{}^{\\color[rgb]{0,0.6,0}\\blacktriangle}"
],
"cells_used": [
"0.32"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2607.14181v1.html",
"sha256": "97ed44759c94b2ae6ba90499eaad1ad517eb72e736607f6fab29cb587fe7efac",
"html_table_id": "S5.T4.3.1",
"html_row_id": "S5.T4.3.1.10",
"raw_cells": [
"Qwen2.5-Coder-7B",
"16 bit",
"FP",
"0.41",
"17281",
"0",
"21",
"670",
"2975",
"2492"
],
"cells_used": [
"0.41"
]
},
"transformation": "Published pass@1 fractions (two decimals) used as-is; retention = quant / FP16",
"note": "Pass@1 is published to two decimals only, so retention is coarse (+/-0.025 absolute). Pre-quantized Hugging Face checkpoints were used where available, otherwise quantized locally with official tools; the exact checkpoint per method is not identified.",
"aggregation": "single benchmark",
"study": "arXiv:2607.14181v1:BCB-Python:Qwen2.5-Coder-7B-Instruct",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GPTQ-style W2A16 (weight-only 2-bit; group size not stated)",
"metric": "GSM8K accuracy, thinking mode, 32K reasoning budget, temperature 0.6",
"full": 0.9583,
"quant_score": 0.9333,
"retention": 0.9739121360742982,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2606.02011v1#S6.T2",
"source_id": "quant-lowbit-reasoning-v1",
"evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.6",
"raw_cells": [
"Qwen3-32B",
"GSM8K",
"95.83",
"93.33",
"-2.50",
"99.77",
"92.49 (-0.84)",
"93.48 (+0.15)",
"92.65 (-0.68)"
],
"cells_used": [
"93.33"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.6",
"raw_cells": [
"Qwen3-32B",
"GSM8K",
"95.83",
"93.33",
"-2.50",
"99.77",
"92.49 (-0.84)",
"93.48 (+0.15)",
"92.65 (-0.68)"
],
"cells_used": [
"95.83"
]
},
"transformation": "Published accuracy percentages / 100; retention = 2-bit / FP16",
"note": "Authors' own runs (A100). 2-bit is an aggressive research setting, not a mainstream GGUF quant; the paper's W4A16 results are shown only in figures.",
"aggregation": "single benchmark",
"study": "arXiv:2606.02011v1:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GPTQ-style W2A16 (weight-only 2-bit; group size not stated)",
"metric": "MATH-500 accuracy, thinking mode, 32K reasoning budget, temperature 0.6",
"full": 0.93,
"quant_score": 0.65,
"retention": 0.6989247311827956,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2606.02011v1#S6.T2",
"source_id": "quant-lowbit-reasoning-v1",
"evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.14",
"raw_cells": [
"Qwen3-32B",
"MATH-500",
"93.00",
"65.00",
"-28.00",
"84.20",
"80.60 (+15.60)",
"73.20 (+8.20)",
"87.20 (+22.20)"
],
"cells_used": [
"65.00"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.14",
"raw_cells": [
"Qwen3-32B",
"MATH-500",
"93.00",
"65.00",
"-28.00",
"84.20",
"80.60 (+15.60)",
"73.20 (+8.20)",
"87.20 (+22.20)"
],
"cells_used": [
"93.00"
]
},
"transformation": "Published accuracy percentages / 100; retention = 2-bit / FP16",
"note": "Authors' own runs (A100). 2-bit is an aggressive research setting, not a mainstream GGUF quant; the paper's W4A16 results are shown only in figures.",
"aggregation": "single benchmark",
"study": "arXiv:2606.02011v1:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GPTQ-style W2A16 (weight-only 2-bit; group size not stated)",
"metric": "GPQA-Diamond accuracy, thinking mode, 32K reasoning budget, temperature 0.6",
"full": 0.6616,
"quant_score": 0.4293,
"retention": 0.6488814993954052,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2606.02011v1#S6.T2",
"source_id": "quant-lowbit-reasoning-v1",
"evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.15",
"raw_cells": [
"Qwen3-32B",
"GPQA-Diamond",
"66.16",
"42.93",
"-23.23",
"80.81",
"48.22 (+5.29)",
"52.53 (+9.60)",
"51.78 (+8.85)"
],
"cells_used": [
"42.93"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.15",
"raw_cells": [
"Qwen3-32B",
"GPQA-Diamond",
"66.16",
"42.93",
"-23.23",
"80.81",
"48.22 (+5.29)",
"52.53 (+9.60)",
"51.78 (+8.85)"
],
"cells_used": [
"66.16"
]
},
"transformation": "Published accuracy percentages / 100; retention = 2-bit / FP16",
"note": "Authors' own runs (A100). 2-bit is an aggressive research setting, not a mainstream GGUF quant; the paper's W4A16 results are shown only in figures.",
"aggregation": "single benchmark",
"study": "arXiv:2606.02011v1:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-32B",
"model_id": "qwen3-32b",
"params_b": 32.8,
"quant": "GPTQ-style W2A16 (weight-only 2-bit; group size not stated)",
"metric": "AIME 2026 accuracy, thinking mode, 32K reasoning budget, temperature 0.6",
"full": 0.7,
"quant_score": 0.1667,
"retention": 0.23814285714285713,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2606.02011v1#S6.T2",
"source_id": "quant-lowbit-reasoning-v1",
"evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.24",
"raw_cells": [
"Qwen3-32B",
"AIME2026",
"70.00",
"16.67",
"-53.33",
"66.67",
"16.67 (+0.00)",
"36.67 (+20.00)",
"40.00 (+23.33)"
],
"cells_used": [
"16.67"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.24",
"raw_cells": [
"Qwen3-32B",
"AIME2026",
"70.00",
"16.67",
"-53.33",
"66.67",
"16.67 (+0.00)",
"36.67 (+20.00)",
"40.00 (+23.33)"
],
"cells_used": [
"70.00"
]
},
"transformation": "Published accuracy percentages / 100; retention = 2-bit / FP16",
"note": "Authors' own runs (A100). 2-bit is an aggressive research setting, not a mainstream GGUF quant; the paper's W4A16 results are shown only in figures.",
"aggregation": "single benchmark",
"study": "arXiv:2606.02011v1:Qwen3-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GPTQ-style W2A16 (weight-only 2-bit; group size not stated)",
"metric": "GSM8K accuracy, thinking mode, 32K reasoning budget, temperature 0.6",
"full": 0.9613,
"quant_score": 0.5269,
"retention": 0.5481119317590762,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2606.02011v1#S6.T2",
"source_id": "quant-lowbit-reasoning-v1",
"evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.18",
"raw_cells": [
"Qwen3-8B",
"GSM8K",
"96.13",
"52.69",
"-43.44",
"27.14",
"65.73 (+13.04)",
"74.45 (+21.76)",
"80.06 (+27.37)"
],
"cells_used": [
"52.69"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.18",
"raw_cells": [
"Qwen3-8B",
"GSM8K",
"96.13",
"52.69",
"-43.44",
"27.14",
"65.73 (+13.04)",
"74.45 (+21.76)",
"80.06 (+27.37)"
],
"cells_used": [
"96.13"
]
},
"transformation": "Published accuracy percentages / 100; retention = 2-bit / FP16",
"note": "Authors' own runs (A100). 2-bit is an aggressive research setting, not a mainstream GGUF quant; the paper's W4A16 results are shown only in figures.",
"aggregation": "single benchmark",
"study": "arXiv:2606.02011v1:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GPTQ-style W2A16 (weight-only 2-bit; group size not stated)",
"metric": "MATH-500 accuracy, thinking mode, 32K reasoning budget, temperature 0.6",
"full": 0.924,
"quant_score": 0.172,
"retention": 0.18614718614718612,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2606.02011v1#S6.T2",
"source_id": "quant-lowbit-reasoning-v1",
"evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.25",
"raw_cells": [
"Qwen3-8B",
"MATH-500",
"92.40",
"17.20",
"-75.20",
"23.60",
"26.40 (+9.20)",
"74.20 (+57.00)",
"76.20 (+59.00)"
],
"cells_used": [
"17.20"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.25",
"raw_cells": [
"Qwen3-8B",
"MATH-500",
"92.40",
"17.20",
"-75.20",
"23.60",
"26.40 (+9.20)",
"74.20 (+57.00)",
"76.20 (+59.00)"
],
"cells_used": [
"92.40"
]
},
"transformation": "Published accuracy percentages / 100; retention = 2-bit / FP16",
"note": "Authors' own runs (A100). 2-bit is an aggressive research setting, not a mainstream GGUF quant; the paper's W4A16 results are shown only in figures.",
"aggregation": "single benchmark",
"study": "arXiv:2606.02011v1:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "GPTQ-style W2A16 (weight-only 2-bit; group size not stated)",
"metric": "GPQA-Diamond accuracy, thinking mode, 32K reasoning budget, temperature 0.6",
"full": 0.5909,
"quant_score": 0.1717,
"retention": 0.2905737011338636,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2606.02011v1#S6.T2",
"source_id": "quant-lowbit-reasoning-v1",
"evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.26",
"raw_cells": [
"Qwen3-8B",
"GPQA-Diamond",
"59.09",
"17.17",
"-41.92",
"31.31",
"17.17 (+0.00)",
"47.98 (+30.81)",
"52.53 (+35.36)"
],
"cells_used": [
"17.17"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2606.02011v1.html",
"sha256": "49b83208d8a9a91de88581d9b366c5e80652c4172d0aa673ea9114c7d1ba3489",
"html_table_id": "S6.T2.17",
"html_row_id": "S6.T2.17.26",
"raw_cells": [
"Qwen3-8B",
"GPQA-Diamond",
"59.09",
"17.17",
"-41.92",
"31.31",
"17.17 (+0.00)",
"47.98 (+30.81)",
"52.53 (+35.36)"
],
"cells_used": [
"59.09"
]
},
"transformation": "Published accuracy percentages / 100; retention = 2-bit / FP16",
"note": "Authors' own runs (A100). 2-bit is an aggressive research setting, not a mainstream GGUF quant; the paper's W4A16 results are shown only in figures.",
"aggregation": "single benchmark",
"study": "arXiv:2606.02011v1:Qwen3-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GPTQ-Int8 (Qwen2.5-7B-Instruct-GPTQ-Int8 checkpoint, as named in Table 1)",
"metric": "Bangla zero-shot accuracy, unweighted mean of 5 benchmarks (Bangla MMLU, CommonsenseQA-BN, OpenBookQA-BN, PIQA-BN, BoolQ-BN)",
"full": 0.5814,
"quant_score": 0.5834,
"retention": 1.0034399724802199,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2608.24615v1#S4.T3",
"source_id": "quant-bangla-ptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2608.24615v1.html",
"sha256": "b00831ce9b9cdedbc9799ef097e5552be66ba29480825356bf7c2daf4f1fb3b2",
"html_table_id": "S4.T3.2.1",
"html_row_id": "S4.T3.2.1.4",
"raw_cells": [
"Qwen (GPTQ-Int8)",
"0.446",
"0.456",
"0.543",
"0.618",
"0.854"
],
"cells_used": [
"0.446",
"0.456",
"0.543",
"0.618",
"0.854"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2608.24615v1.html",
"sha256": "b00831ce9b9cdedbc9799ef097e5552be66ba29480825356bf7c2daf4f1fb3b2",
"html_table_id": "S4.T2.2.1",
"html_row_id": "S4.T2.2.1.4",
"raw_cells": [
"Qwen-2.5-7B",
"0.440",
"0.451",
"0.545",
"0.617",
"0.854"
],
"cells_used": [
"0.440",
"0.451",
"0.545",
"0.617",
"0.854"
]
},
"transformation": "Published accuracies are already fractions; five cells averaged by whatmodel.sh (unweighted; benchmark sizes differ); retention = mean_quant / mean_FP16",
"note": "Bangla-language tasks only; not a published average.",
"aggregation": "mean of 5 benchmarks (computed)",
"study": "arXiv:2608.24615v1:Qwen2.5-7B-Instruct",
"artifact": "Qwen2.5-7B-Instruct-GPTQ-Int8",
"artifact_revision": null
},
{
"model": "Qwen3-8B (instruct, thinking disabled)",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 (nvidia/Qwen3-8B-NVFP4; W4A4 FP4)",
"metric": "MMLU 5-shot (500-example subset; lm-eval v0.4.9, vLLM 0.12)",
"full": 0.7729,
"quant_score": 0.7509,
"retention": 0.9715357743563203,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.09527v1#S4.T1",
"source_id": "quant-consumer-blackwell-v1",
"evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T1.4",
"html_row_id": "S4.T1.4.2",
"raw_cells": [
"Qwen3-8B",
"8.2",
"NVFP4",
"0.7509 ± 0.0038"
],
"cells_used": [
"0.7509 ± 0.0038"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T1.4",
"html_row_id": "S4.T1.4.3",
"raw_cells": [
"Qwen3-8B",
"8.2",
"BF16 (baseline)",
"0.7729 ± 0.0036"
],
"cells_used": [
"0.7729 ± 0.0036"
]
},
"transformation": "Published accuracy fractions used as-is (reported +/- 95% CI dropped); retention = quant / BF16",
"note": "Authors' own lm-eval runs on consumer Blackwell GPUs; NVFP4 quantizes weights and activations. Reported 95% CIs overlap only partly; single-run evaluation.",
"aggregation": "single benchmark",
"study": "arXiv:2601.09527v1:Qwen3-8B (instruct, thinking disabled)",
"artifact": "nvidia/Qwen3-8B-NVFP4",
"artifact_revision": null
},
{
"model": "Gemma-3-12B-it",
"model_id": "gemma-3-12b",
"params_b": 12.2,
"quant": "W4A16 (RedHatAI gemma-3-12b-it-quantized.w4a16)",
"metric": "MMLU 5-shot (500-example subset; lm-eval v0.4.9, vLLM 0.12)",
"full": 0.6202,
"quant_score": 0.5904,
"retention": 0.9519509835536925,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.09527v1#S4.T1",
"source_id": "quant-consumer-blackwell-v1",
"evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T1.4",
"html_row_id": "S4.T1.4.4",
"raw_cells": [
"Gemma3-12B",
"12",
"W4A16",
"0.5904 ± 0.0043"
],
"cells_used": [
"0.5904 ± 0.0043"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T1.4",
"html_row_id": "S4.T1.4.6",
"raw_cells": [
"Gemma3-12B",
"12",
"BF16 (baseline)",
"0.6202 ± 0.0042"
],
"cells_used": [
"0.6202 ± 0.0042"
]
},
"transformation": "Published accuracy fractions used as-is (reported +/- 95% CI dropped); retention = quant / BF16",
"note": "Authors' own lm-eval runs on consumer Blackwell GPUs; NVFP4 quantizes weights and activations. Reported 95% CIs overlap only partly; single-run evaluation.",
"aggregation": "single benchmark",
"study": "arXiv:2601.09527v1:Gemma-3-12B-it",
"artifact": "RedHatAI/gemma-3-12b-it-quantized.w4a16",
"artifact_revision": null
},
{
"model": "Gemma-3-12B-it",
"model_id": "gemma-3-12b",
"params_b": 12.2,
"quant": "NVFP4 (weights and activations; checkpoint not named)",
"metric": "MMLU 5-shot (500-example subset; lm-eval v0.4.9, vLLM 0.12)",
"full": 0.6202,
"quant_score": 0.5795,
"retention": 0.934376007739439,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.09527v1#S4.T1",
"source_id": "quant-consumer-blackwell-v1",
"evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T1.4",
"html_row_id": "S4.T1.4.5",
"raw_cells": [
"Gemma3-12B",
"12",
"NVFP4",
"0.5795 ± 0.0043"
],
"cells_used": [
"0.5795 ± 0.0043"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T1.4",
"html_row_id": "S4.T1.4.6",
"raw_cells": [
"Gemma3-12B",
"12",
"BF16 (baseline)",
"0.6202 ± 0.0042"
],
"cells_used": [
"0.6202 ± 0.0042"
]
},
"transformation": "Published accuracy fractions used as-is (reported +/- 95% CI dropped); retention = quant / BF16",
"note": "Authors' own lm-eval runs on consumer Blackwell GPUs; NVFP4 quantizes weights and activations. Reported 95% CIs overlap only partly; single-run evaluation.",
"aggregation": "single benchmark",
"study": "arXiv:2601.09527v1:Gemma-3-12B-it",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen3-8B (instruct, thinking disabled)",
"model_id": "qwen3-8b",
"params_b": 8.19,
"quant": "NVFP4 (nvidia/Qwen3-8B-NVFP4; W4A4 FP4)",
"metric": "GSM8K chain-of-thought exact match (1,319 problems; lm-eval v0.4.9, vLLM 0.12)",
"full": 0.8901,
"quant_score": 0.8711,
"retention": 0.9786540838108078,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.09527v1#S4.T2",
"source_id": "quant-consumer-blackwell-v1",
"evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T2.4",
"html_row_id": "S4.T2.4.2",
"raw_cells": [
"Qwen3-8B",
"8.2",
"NVFP4",
"0.8711 ± 0.0092"
],
"cells_used": [
"0.8711 ± 0.0092"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T2.4",
"html_row_id": "S4.T2.4.3",
"raw_cells": [
"Qwen3-8B",
"8.2",
"BF16 (baseline)",
"0.8901 ± 0.0086"
],
"cells_used": [
"0.8901 ± 0.0086"
]
},
"transformation": "Published accuracy fractions used as-is (reported +/- 95% CI dropped); retention = quant / BF16",
"note": "Authors' own lm-eval runs on consumer Blackwell GPUs; NVFP4 quantizes weights and activations. Reported 95% CIs overlap only partly; single-run evaluation.",
"aggregation": "single benchmark",
"study": "arXiv:2601.09527v1:Qwen3-8B (instruct, thinking disabled)",
"artifact": "nvidia/Qwen3-8B-NVFP4",
"artifact_revision": null
},
{
"model": "Gemma-3-12B-it",
"model_id": "gemma-3-12b",
"params_b": 12.2,
"quant": "W4A16 (RedHatAI gemma-3-12b-it-quantized.w4a16)",
"metric": "GSM8K chain-of-thought exact match (1,319 problems; lm-eval v0.4.9, vLLM 0.12)",
"full": 0.8249,
"quant_score": 0.8203,
"retention": 0.9944235664929083,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.09527v1#S4.T2",
"source_id": "quant-consumer-blackwell-v1",
"evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T2.4",
"html_row_id": "S4.T2.4.4",
"raw_cells": [
"Gemma3-12B",
"12",
"W4A16",
"0.8203 ± 0.0106"
],
"cells_used": [
"0.8203 ± 0.0106"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T2.4",
"html_row_id": "S4.T2.4.6",
"raw_cells": [
"Gemma3-12B",
"12",
"BF16 (baseline)",
"0.8249 ± 0.0105"
],
"cells_used": [
"0.8249 ± 0.0105"
]
},
"transformation": "Published accuracy fractions used as-is (reported +/- 95% CI dropped); retention = quant / BF16",
"note": "Authors' own lm-eval runs on consumer Blackwell GPUs; NVFP4 quantizes weights and activations. Reported 95% CIs overlap only partly; single-run evaluation.",
"aggregation": "single benchmark",
"study": "arXiv:2601.09527v1:Gemma-3-12B-it",
"artifact": "RedHatAI/gemma-3-12b-it-quantized.w4a16",
"artifact_revision": null
},
{
"model": "Gemma-3-12B-it",
"model_id": "gemma-3-12b",
"params_b": 12.2,
"quant": "NVFP4 (weights and activations; checkpoint not named)",
"metric": "GSM8K chain-of-thought exact match (1,319 problems; lm-eval v0.4.9, vLLM 0.12)",
"full": 0.8249,
"quant_score": 0.7892,
"retention": 0.9567220269123531,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.09527v1#S4.T2",
"source_id": "quant-consumer-blackwell-v1",
"evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T2.4",
"html_row_id": "S4.T2.4.5",
"raw_cells": [
"Gemma3-12B",
"12",
"NVFP4",
"0.7892 ± 0.0112"
],
"cells_used": [
"0.7892 ± 0.0112"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.09527v1.html",
"sha256": "c756e75a163df79afd0b258d9deb5aa2d56ffb3a21d87c1240c70e97525bace4",
"html_table_id": "S4.T2.4",
"html_row_id": "S4.T2.4.6",
"raw_cells": [
"Gemma3-12B",
"12",
"BF16 (baseline)",
"0.8249 ± 0.0105"
],
"cells_used": [
"0.8249 ± 0.0105"
]
},
"transformation": "Published accuracy fractions used as-is (reported +/- 95% CI dropped); retention = quant / BF16",
"note": "Authors' own lm-eval runs on consumer Blackwell GPUs; NVFP4 quantizes weights and activations. Reported 95% CIs overlap only partly; single-run evaluation.",
"aggregation": "single benchmark",
"study": "arXiv:2601.09527v1:Gemma-3-12B-it",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.818,
"quant_score": 0.804,
"retention": 0.9828850855745722,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.10",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation & 4-bit AWQ",
"63.4",
"78.5",
"99.3",
"80.4",
"(10.7, 19.23)"
],
"cells_used": [
"80.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.7",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation",
"65.6",
"79.8",
"99.9",
"81.8",
"(13.3, 21.57)"
],
"cells_used": [
"81.8"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.656,
"quant_score": 0.634,
"retention": 0.9664634146341463,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.10",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation & 4-bit AWQ",
"63.4",
"78.5",
"99.3",
"80.4",
"(10.7, 19.23)"
],
"cells_used": [
"63.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.7",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation",
"65.6",
"79.8",
"99.9",
"81.8",
"(13.3, 21.57)"
],
"cells_used": [
"65.6"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.818,
"quant_score": 0.812,
"retention": 0.9926650366748168,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.11",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation & 4-bit GPTQ",
"66.7",
"77.0",
"99.9",
"81.2",
"(10.3, 18.17)"
],
"cells_used": [
"81.2"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.7",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation",
"65.6",
"79.8",
"99.9",
"81.8",
"(13.3, 21.57)"
],
"cells_used": [
"81.8"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.656,
"quant_score": 0.667,
"retention": 1.016768292682927,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.11",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation & 4-bit GPTQ",
"66.7",
"77.0",
"99.9",
"81.2",
"(10.3, 18.17)"
],
"cells_used": [
"66.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.7",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation",
"65.6",
"79.8",
"99.9",
"81.8",
"(13.3, 21.57)"
],
"cells_used": [
"65.6"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.818,
"quant_score": 0.726,
"retention": 0.8875305623471883,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.13",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation & 3-bit GPTQ",
"46.7",
"71.8",
"99.3",
"72.6",
"(4.7, 11.92)"
],
"cells_used": [
"72.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.7",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation",
"65.6",
"79.8",
"99.9",
"81.8",
"(13.3, 21.57)"
],
"cells_used": [
"81.8"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-70B",
"model_id": "deepseek-r1-distill-llama-70b",
"params_b": 70,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.656,
"quant_score": 0.467,
"retention": 0.711890243902439,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.13",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation & 3-bit GPTQ",
"46.7",
"71.8",
"99.3",
"72.6",
"(4.7, 11.92)"
],
"cells_used": [
"46.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.7",
"raw_cells": [
"R1-Distill-Llama",
"70B",
"Distillation",
"65.6",
"79.8",
"99.9",
"81.8",
"(13.3, 21.57)"
],
"cells_used": [
"65.6"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-70B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.822,
"quant_score": 0.831,
"retention": 1.010948905109489,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.17",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation & 4-bit AWQ",
"67.8",
"82.3",
"99.1",
"83.1",
"(3.3, 10.28)"
],
"cells_used": [
"83.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.15",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation",
"64.4",
"82.3",
"99.9",
"82.2",
"(2.7, 10.95)"
],
"cells_used": [
"82.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.644,
"quant_score": 0.678,
"retention": 1.0527950310559007,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.17",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation & 4-bit AWQ",
"67.8",
"82.3",
"99.1",
"83.1",
"(3.3, 10.28)"
],
"cells_used": [
"67.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.15",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation",
"64.4",
"82.3",
"99.9",
"82.2",
"(2.7, 10.95)"
],
"cells_used": [
"64.4"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.822,
"quant_score": 0.83,
"retention": 1.0097323600973236,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.18",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation & 4-bit GPTQ",
"68.9",
"80.6",
"99.6",
"83.0",
"(4.0, 11.78)"
],
"cells_used": [
"83.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.15",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation",
"64.4",
"82.3",
"99.9",
"82.2",
"(2.7, 10.95)"
],
"cells_used": [
"82.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.644,
"quant_score": 0.689,
"retention": 1.0698757763975155,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.18",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation & 4-bit GPTQ",
"68.9",
"80.6",
"99.6",
"83.0",
"(4.0, 11.78)"
],
"cells_used": [
"68.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.15",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation",
"64.4",
"82.3",
"99.9",
"82.2",
"(2.7, 10.95)"
],
"cells_used": [
"64.4"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.822,
"quant_score": 0.725,
"retention": 0.8819951338199513,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.21",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation & 3-bit GPTQ",
"44.4",
"74.2",
"98.9",
"72.5",
"(4.0, 11.55)"
],
"cells_used": [
"72.5"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.15",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation",
"64.4",
"82.3",
"99.9",
"82.2",
"(2.7, 10.95)"
],
"cells_used": [
"82.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-32B",
"model_id": "deepseek-r1-distill-qwen-32b",
"params_b": 32.8,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.644,
"quant_score": 0.444,
"retention": 0.6894409937888198,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.21",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation & 3-bit GPTQ",
"44.4",
"74.2",
"98.9",
"72.5",
"(4.0, 11.55)"
],
"cells_used": [
"44.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.15",
"raw_cells": [
"R1-Distill-Qwen",
"32B",
"Distillation",
"64.4",
"82.3",
"99.9",
"82.2",
"(2.7, 10.95)"
],
"cells_used": [
"64.4"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-32B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.652,
"quant_score": 0.666,
"retention": 1.021472392638037,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.27",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation & 4-bit AWQ",
"47.8",
"68.0",
"84.0",
"66.6",
"(0.3, 5.05)"
],
"cells_used": [
"66.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.24",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation",
"42.2",
"71.9",
"81.5",
"65.2",
"(0.0, 4.43)"
],
"cells_used": [
"65.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.422,
"quant_score": 0.478,
"retention": 1.132701421800948,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.27",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation & 4-bit AWQ",
"47.8",
"68.0",
"84.0",
"66.6",
"(0.3, 5.05)"
],
"cells_used": [
"47.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.24",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation",
"42.2",
"71.9",
"81.5",
"65.2",
"(0.0, 4.43)"
],
"cells_used": [
"42.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.652,
"quant_score": 0.581,
"retention": 0.8911042944785276,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.28",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation & 4-bit GPTQ",
"42.2",
"66.2",
"65.9",
"58.1",
"(0.3, 4.68)"
],
"cells_used": [
"58.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.24",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation",
"42.2",
"71.9",
"81.5",
"65.2",
"(0.0, 4.43)"
],
"cells_used": [
"65.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.422,
"quant_score": 0.422,
"retention": 1,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.28",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation & 4-bit GPTQ",
"42.2",
"66.2",
"65.9",
"58.1",
"(0.3, 4.68)"
],
"cells_used": [
"42.2"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.24",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation",
"42.2",
"71.9",
"81.5",
"65.2",
"(0.0, 4.43)"
],
"cells_used": [
"42.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.652,
"quant_score": 0.478,
"retention": 0.7331288343558282,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.31",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation & 3-bit GPTQ",
"11.1",
"65.0",
"67.3",
"47.8",
"(0.0, 2.89)"
],
"cells_used": [
"47.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.24",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation",
"42.2",
"71.9",
"81.5",
"65.2",
"(0.0, 4.43)"
],
"cells_used": [
"65.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Llama-8B",
"model_id": "deepseek-r1-distill-llama-8b",
"params_b": 8.03,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.422,
"quant_score": 0.111,
"retention": 0.26303317535545023,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.31",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation & 3-bit GPTQ",
"11.1",
"65.0",
"67.3",
"47.8",
"(0.0, 2.89)"
],
"cells_used": [
"11.1"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.24",
"raw_cells": [
"R1-Distill-Llama",
"8B",
"Distillation",
"42.2",
"71.9",
"81.5",
"65.2",
"(0.0, 4.43)"
],
"cells_used": [
"42.2"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Llama-8B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.668,
"quant_score": 0.657,
"retention": 0.9835329341317365,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.35",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation & 4-bit AWQ",
"46.6",
"75.5",
"74.9",
"65.7",
"(0.0, 3.14)"
],
"cells_used": [
"65.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.34",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation",
"46.7",
"78.0",
"75.6",
"66.8",
"(0.0, 3.57)"
],
"cells_used": [
"66.8"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "AWQ 4-bit (AutoAWQ; vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.467,
"quant_score": 0.466,
"retention": 0.9978586723768736,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.35",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation & 4-bit AWQ",
"46.6",
"75.5",
"74.9",
"65.7",
"(0.0, 3.14)"
],
"cells_used": [
"46.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.34",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation",
"46.7",
"78.0",
"75.6",
"66.8",
"(0.0, 3.57)"
],
"cells_used": [
"46.7"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.668,
"quant_score": 0.607,
"retention": 0.9086826347305389,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.36",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation & 4-bit GPTQ",
"38.9",
"72.9",
"70.3",
"60.7",
"(1.0, 4.27)"
],
"cells_used": [
"60.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.34",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation",
"46.7",
"78.0",
"75.6",
"66.8",
"(0.0, 3.57)"
],
"cells_used": [
"66.8"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "GPTQ 4-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.467,
"quant_score": 0.389,
"retention": 0.8329764453961456,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.36",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation & 4-bit GPTQ",
"38.9",
"72.9",
"70.3",
"60.7",
"(1.0, 4.27)"
],
"cells_used": [
"38.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.34",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation",
"46.7",
"78.0",
"75.6",
"66.8",
"(0.0, 3.57)"
],
"cells_used": [
"46.7"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "Average of AIME 2024, FOLIO and Temporal accuracy (published 'Avg'; 3 passes each; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.668,
"quant_score": 0.384,
"retention": 0.5748502994011976,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.39",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation & 3-bit GPTQ",
"17.8",
"65.7",
"31.7",
"38.4",
"(0.0, 3.12)"
],
"cells_used": [
"38.4"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.34",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation",
"46.7",
"78.0",
"75.6",
"66.8",
"(0.0, 3.57)"
],
"cells_used": [
"66.8"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "published suite average of 3 tasks",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "DeepSeek-R1-Distill-Qwen-7B",
"model_id": "deepseek-r1-distill-qwen-7b",
"params_b": 7.62,
"quant": "GPTQ 3-bit (vLLM)",
"metric": "AIME 2024 accuracy (mean of 3 passes; vLLM, T=0.6, top-p 0.95, 32768 max tokens)",
"full": 0.467,
"quant_score": 0.178,
"retention": 0.38115631691648816,
"baseline_quant": "unquantized distilled checkpoint run on vLLM (same study; dtype not stated)",
"source": "https://arxiv.org/html/2504.02010v3#S3.T1",
"source_id": "quant-reasoning-compression-v3",
"evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.39",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation & 3-bit GPTQ",
"17.8",
"65.7",
"31.7",
"38.4",
"(0.0, 3.12)"
],
"cells_used": [
"17.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2504.02010v3.html",
"sha256": "500b37b4b677dfe5246f304d6238a797883851282f6b383729414b89b0613b57",
"html_table_id": "S3.T1.6.1",
"html_row_id": "S3.T1.6.1.34",
"raw_cells": [
"R1-Distill-Qwen",
"7B",
"Distillation",
"46.7",
"78.0",
"75.6",
"66.8",
"(0.0, 3.57)"
],
"cells_used": [
"46.7"
]
},
"transformation": "Published percentages / 100; retention = quantized / unquantized distilled model; Avg column is the paper's own average",
"note": "Authors' own runs of the public R1-distilled checkpoints. Group size/calibration for GPTQ/AWQ not stated in the table. AIME 2024 has 30 problems, so single-benchmark retention is noisy. The paper's DeepSeek-R1 dynamic-quant rows use a DeepSeek API baseline and are excluded.",
"aggregation": "single benchmark",
"study": "arXiv:2504.02010v3:DeepSeek-R1-Distill-Qwen-7B",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 PTQ (post-training, W4A4 FP4)",
"metric": "MATH-500 accuracy (NVIDIA evaluation settings)",
"full": 0.978,
"quant_score": 0.972,
"retention": 0.9938650306748467,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.7",
"raw_cells": [
"NVFP4 PTQ",
"97.2",
"69.8",
"59.0",
"89.8"
],
"cells_used": [
"97.2"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"97.8"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 PTQ",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 PTQ (post-training, W4A4 FP4)",
"metric": "AIME 2025 accuracy (NVIDIA evaluation settings)",
"full": 0.711,
"quant_score": 0.698,
"retention": 0.9817158931082982,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.7",
"raw_cells": [
"NVFP4 PTQ",
"97.2",
"69.8",
"59.0",
"89.8"
],
"cells_used": [
"69.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"71.1"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 PTQ",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 PTQ (post-training, W4A4 FP4)",
"metric": "GPQA-Diamond accuracy (NVIDIA evaluation settings)",
"full": 0.64,
"quant_score": 0.59,
"retention": 0.9218749999999999,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.7",
"raw_cells": [
"NVFP4 PTQ",
"97.2",
"69.8",
"59.0",
"89.8"
],
"cells_used": [
"59.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"64.0"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 PTQ",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 PTQ (post-training, W4A4 FP4)",
"metric": "IFEval (instruction-level) accuracy (NVIDIA evaluation settings)",
"full": 0.903,
"quant_score": 0.898,
"retention": 0.9944629014396457,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.7",
"raw_cells": [
"NVFP4 PTQ",
"97.2",
"69.8",
"59.0",
"89.8"
],
"cells_used": [
"89.8"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"90.3"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 PTQ",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 QAD (quantization-aware distillation)",
"metric": "MATH-500 accuracy (NVIDIA evaluation settings)",
"full": 0.978,
"quant_score": 0.972,
"retention": 0.9938650306748467,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.9",
"raw_cells": [
"NVFP4 QAD",
"97.2",
"71.5",
"62.7",
"89.3"
],
"cells_used": [
"97.2"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"97.8"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 QAD",
"artifact": "NVIDIA NVFP4 checkpoint of Nemotron Nano 9B V2 (linked from the paper; revision not pinned)",
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 QAD (quantization-aware distillation)",
"metric": "AIME 2025 accuracy (NVIDIA evaluation settings)",
"full": 0.711,
"quant_score": 0.715,
"retention": 1.0056258790436006,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.9",
"raw_cells": [
"NVFP4 QAD",
"97.2",
"71.5",
"62.7",
"89.3"
],
"cells_used": [
"71.5"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"71.1"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 QAD",
"artifact": "NVIDIA NVFP4 checkpoint of Nemotron Nano 9B V2 (linked from the paper; revision not pinned)",
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 QAD (quantization-aware distillation)",
"metric": "GPQA-Diamond accuracy (NVIDIA evaluation settings)",
"full": 0.64,
"quant_score": 0.627,
"retention": 0.9796874999999999,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.9",
"raw_cells": [
"NVFP4 QAD",
"97.2",
"71.5",
"62.7",
"89.3"
],
"cells_used": [
"62.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"64.0"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 QAD",
"artifact": "NVIDIA NVFP4 checkpoint of Nemotron Nano 9B V2 (linked from the paper; revision not pinned)",
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-Nano-9B-v2",
"model_id": "nemotron-nano-9b-v2",
"params_b": 8.89,
"quant": "NVFP4 QAD (quantization-aware distillation)",
"metric": "IFEval (instruction-level) accuracy (NVIDIA evaluation settings)",
"full": 0.903,
"quant_score": 0.893,
"retention": 0.9889258028792912,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T2",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.9",
"raw_cells": [
"NVFP4 QAD",
"97.2",
"71.5",
"62.7",
"89.3"
],
"cells_used": [
"89.3"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T2.5",
"html_row_id": "S3.T2.5.6",
"raw_cells": [
"Nemotron Nano V2",
"BF16",
"97.8",
"71.1",
"64.0",
"90.3"
],
"cells_used": [
"90.3"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-Nano-9B-v2:NVFP4 QAD",
"artifact": "NVIDIA NVFP4 checkpoint of Nemotron Nano 9B V2 (linked from the paper; revision not pinned)",
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-3-Nano-30B-A3B",
"model_id": "nemotron-3-nano-30b-a3b",
"params_b": 31.6,
"quant": "NVFP4 PTQ (post-training, W4A4 FP4)",
"metric": "AIME 2025 accuracy (NVIDIA evaluation settings)",
"full": 0.891,
"quant_score": 0.85,
"retention": 0.9539842873176206,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T3",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.3",
"raw_cells": [
"NVFP4 PTQ",
"31.3",
"85.0",
"71.6",
"68.9",
"30.5"
],
"cells_used": [
"85.0"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.2",
"raw_cells": [
"BF16",
"35.9",
"89.1",
"73.0",
"72.1",
"33.0"
],
"cells_used": [
"89.1"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints. Table 3(a) is captioned 'Nemotron 3 Nano'; Appendix Table 11 names it Nemotron 3 Nano 30B-A3B with identical BF16/PTQ cells.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-3-Nano-30B-A3B:NVFP4 PTQ",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-3-Nano-30B-A3B",
"model_id": "nemotron-3-nano-30b-a3b",
"params_b": 31.6,
"quant": "NVFP4 PTQ (post-training, W4A4 FP4)",
"metric": "GPQA-Diamond accuracy (NVIDIA evaluation settings)",
"full": 0.73,
"quant_score": 0.716,
"retention": 0.9808219178082191,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T3",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.3",
"raw_cells": [
"NVFP4 PTQ",
"31.3",
"85.0",
"71.6",
"68.9",
"30.5"
],
"cells_used": [
"71.6"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.2",
"raw_cells": [
"BF16",
"35.9",
"89.1",
"73.0",
"72.1",
"33.0"
],
"cells_used": [
"73.0"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints. Table 3(a) is captioned 'Nemotron 3 Nano'; Appendix Table 11 names it Nemotron 3 Nano 30B-A3B with identical BF16/PTQ cells.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-3-Nano-30B-A3B:NVFP4 PTQ",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-3-Nano-30B-A3B",
"model_id": "nemotron-3-nano-30b-a3b",
"params_b": 31.6,
"quant": "NVFP4 PTQ (post-training, W4A4 FP4)",
"metric": "LiveCodeBench v5 accuracy (NVIDIA evaluation settings)",
"full": 0.721,
"quant_score": 0.689,
"retention": 0.9556171983356448,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T3",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.3",
"raw_cells": [
"NVFP4 PTQ",
"31.3",
"85.0",
"71.6",
"68.9",
"30.5"
],
"cells_used": [
"68.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.2",
"raw_cells": [
"BF16",
"35.9",
"89.1",
"73.0",
"72.1",
"33.0"
],
"cells_used": [
"72.1"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints. Table 3(a) is captioned 'Nemotron 3 Nano'; Appendix Table 11 names it Nemotron 3 Nano 30B-A3B with identical BF16/PTQ cells.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-3-Nano-30B-A3B:NVFP4 PTQ",
"artifact": null,
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-3-Nano-30B-A3B",
"model_id": "nemotron-3-nano-30b-a3b",
"params_b": 31.6,
"quant": "NVFP4 QAD (quantization-aware distillation)",
"metric": "AIME 2025 accuracy (NVIDIA evaluation settings)",
"full": 0.891,
"quant_score": 0.879,
"retention": 0.9865319865319865,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T3",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.5",
"raw_cells": [
"NVFP4 QAD",
"34.3",
"87.9",
"72.7",
"68.9",
"32.3"
],
"cells_used": [
"87.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.2",
"raw_cells": [
"BF16",
"35.9",
"89.1",
"73.0",
"72.1",
"33.0"
],
"cells_used": [
"89.1"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints. Table 3(a) is captioned 'Nemotron 3 Nano'; Appendix Table 11 names it Nemotron 3 Nano 30B-A3B with identical BF16/PTQ cells.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-3-Nano-30B-A3B:NVFP4 QAD",
"artifact": "NVIDIA NVFP4 checkpoint of Nemotron 3 Nano 30B-A3B (linked from the paper; revision not pinned)",
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-3-Nano-30B-A3B",
"model_id": "nemotron-3-nano-30b-a3b",
"params_b": 31.6,
"quant": "NVFP4 QAD (quantization-aware distillation)",
"metric": "GPQA-Diamond accuracy (NVIDIA evaluation settings)",
"full": 0.73,
"quant_score": 0.727,
"retention": 0.9958904109589041,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T3",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.5",
"raw_cells": [
"NVFP4 QAD",
"34.3",
"87.9",
"72.7",
"68.9",
"32.3"
],
"cells_used": [
"72.7"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.2",
"raw_cells": [
"BF16",
"35.9",
"89.1",
"73.0",
"72.1",
"33.0"
],
"cells_used": [
"73.0"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints. Table 3(a) is captioned 'Nemotron 3 Nano'; Appendix Table 11 names it Nemotron 3 Nano 30B-A3B with identical BF16/PTQ cells.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-3-Nano-30B-A3B:NVFP4 QAD",
"artifact": "NVIDIA NVFP4 checkpoint of Nemotron 3 Nano 30B-A3B (linked from the paper; revision not pinned)",
"artifact_revision": null
},
{
"model": "NVIDIA-Nemotron-3-Nano-30B-A3B",
"model_id": "nemotron-3-nano-30b-a3b",
"params_b": 31.6,
"quant": "NVFP4 QAD (quantization-aware distillation)",
"metric": "LiveCodeBench v5 accuracy (NVIDIA evaluation settings)",
"full": 0.721,
"quant_score": 0.689,
"retention": 0.9556171983356448,
"baseline_quant": "BF16 (same study)",
"source": "https://arxiv.org/html/2601.20088v3#S3.T3",
"source_id": "quant-nvfp4-qad-v3",
"evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.5",
"raw_cells": [
"NVFP4 QAD",
"34.3",
"87.9",
"72.7",
"68.9",
"32.3"
],
"cells_used": [
"68.9"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2601.20088v3.html",
"sha256": "e5ebc7949a3aef349273417ebbddd9df45bfe63957f1024f883a64649626e251",
"html_table_id": "S3.T3.6",
"html_row_id": "S3.T3.6.2",
"raw_cells": [
"BF16",
"35.9",
"89.1",
"73.0",
"72.1",
"33.0"
],
"cells_used": [
"72.1"
]
},
"transformation": "Published percentages / 100; retention = NVFP4 / BF16",
"note": "NVIDIA's own evaluation of its own models (lab-run); the paper notes benchmark numbers differ from the model cards because different evaluation tools/settings were used. PTQ = post-training NVFP4 (weights and activations); QAD = the paper's quantization-aware distillation, the method behind NVIDIA's released NVFP4 checkpoints. Table 3(a) is captioned 'Nemotron 3 Nano'; Appendix Table 11 names it Nemotron 3 Nano 30B-A3B with identical BF16/PTQ cells.",
"aggregation": "single benchmark",
"study": "arXiv:2601.20088v3:Nemotron-3-Nano-30B-A3B:NVFP4 QAD",
"artifact": "NVIDIA NVFP4 checkpoint of Nemotron 3 Nano 30B-A3B (linked from the paper; revision not pinned)",
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GPTQ INT8 (AutoGPTQ 0.7.1, group size 128, no act-order, 128 WikiText-2 calibration samples)",
"metric": "Clinical accuracy, unweighted mean of 4 benchmarks (MedQA n=1273, MedMCQA, Med-HALT, HealthBench risk-stratified n=150); greedy decoding; Transformers on H100",
"full": 0.4355,
"quant_score": 0.4345,
"retention": 0.9977037887485649,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2609.22216v1#Sx5.T1",
"source_id": "quant-clinical-gptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2609.22216v1.html",
"sha256": "65b7c3323a9a7b046db36c22067e232fd6ad2b5673fb45bca6ef9338caeec187",
"html_table_id": "Sx5.T1.1",
"html_row_id": "Sx5.T1.1.7",
"raw_cells": [
"Qwen2.5-7B∘\\,\\circ",
"MedQA",
"0.596",
"[0.57, 0.62]",
"0.596",
"+0.0+0.0",
"0.589",
"−1.2-1.2"
],
"rows": [
"Sx5.T1.1.7",
"Sx5.T1.1.8",
"Sx5.T1.1.9",
"Sx5.T1.1.10"
],
"cells_used": [
"0.596",
"0.299",
"0.613",
"0.230"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2609.22216v1.html",
"sha256": "65b7c3323a9a7b046db36c22067e232fd6ad2b5673fb45bca6ef9338caeec187",
"html_table_id": "Sx5.T1.1",
"html_row_id": "Sx5.T1.1.7",
"raw_cells": [
"Qwen2.5-7B∘\\,\\circ",
"MedQA",
"0.596",
"[0.57, 0.62]",
"0.596",
"+0.0+0.0",
"0.589",
"−1.2-1.2"
],
"rows": [
"Sx5.T1.1.7",
"Sx5.T1.1.8",
"Sx5.T1.1.9",
"Sx5.T1.1.10"
],
"cells_used": [
"0.596",
"0.297",
"0.613",
"0.236"
]
},
"transformation": "Published accuracy fractions; four benchmark cells averaged by whatmodel.sh (unweighted; not a published average); retention = mean_quant / mean_FP16",
"note": "Domain-specific (clinical) benchmarks. The paper links Qwen/Qwen2.5-7B-Instruct. Per-benchmark deltas range from +0.7% to -3.0%.",
"aggregation": "mean of 4 benchmarks (computed)",
"study": "arXiv:2609.22216v1:Qwen2.5-7B-Instruct",
"artifact": null,
"artifact_revision": null
},
{
"model": "Qwen2.5-7B-Instruct",
"model_id": "qwen2-5-7b",
"params_b": 7.62,
"quant": "GPTQ INT4 (AutoGPTQ 0.7.1, group size 128, no act-order, 128 WikiText-2 calibration samples)",
"metric": "Clinical accuracy, unweighted mean of 4 benchmarks (MedQA n=1273, MedMCQA, Med-HALT, HealthBench risk-stratified n=150); greedy decoding; Transformers on H100",
"full": 0.4355,
"quant_score": 0.43075,
"retention": 0.9890929965556831,
"baseline_quant": "FP16 (same study)",
"source": "https://arxiv.org/html/2609.22216v1#Sx5.T1",
"source_id": "quant-clinical-gptq-v1",
"evidence": {
"file": "speed-evidence/arxiv/2609.22216v1.html",
"sha256": "65b7c3323a9a7b046db36c22067e232fd6ad2b5673fb45bca6ef9338caeec187",
"html_table_id": "Sx5.T1.1",
"html_row_id": "Sx5.T1.1.7",
"raw_cells": [
"Qwen2.5-7B∘\\,\\circ",
"MedQA",
"0.596",
"[0.57, 0.62]",
"0.596",
"+0.0+0.0",
"0.589",
"−1.2-1.2"
],
"rows": [
"Sx5.T1.1.7",
"Sx5.T1.1.8",
"Sx5.T1.1.9",
"Sx5.T1.1.10"
],
"cells_used": [
"0.589",
"0.292",
"0.613",
"0.229"
]
},
"baseline_evidence": {
"file": "speed-evidence/arxiv/2609.22216v1.html",
"sha256": "65b7c3323a9a7b046db36c22067e232fd6ad2b5673fb45bca6ef9338caeec187",
"html_table_id": "Sx5.T1.1",
"html_row_id": "Sx5.T1.1.7",
"raw_cells": [
"Qwen2.5-7B∘\\,\\circ",
"MedQA",
"0.596",
"[0.57, 0.62]",
"0.596",
"+0.0+0.0",
"0.589",
"−1.2-1.2"
],
"rows": [
"Sx5.T1.1.7",
"Sx5.T1.1.8",
"Sx5.T1.1.9",
"Sx5.T1.1.10"
],
"cells_used": [
"0.596",
"0.297",
"0.613",
"0.236"
]
},
"transformation": "Published accuracy fractions; four benchmark cells averaged by whatmodel.sh (unweighted; not a published average); retention = mean_quant / mean_FP16",
"note": "Domain-specific (clinical) benchmarks. The paper links Qwen/Qwen2.5-7B-Instruct. Per-benchmark deltas range from +0.7% to -3.0%.",
"aggregation": "mean of 4 benchmarks (computed)",
"study": "arXiv:2609.22216v1:Qwen2.5-7B-Instruct",
"artifact": null,
"artifact_revision": null
}
],
"memory": [
{
"model": "gemma-4-e2b",
"model_label": "E2B",
"quant": "mobile (mixed int2/int4 weights, int8 activations)",
"bf16_weights_as_published": "4.6",
"quantized_weights_as_published": "0.8",
"int8_kv_cache_at_32k_as_published": "+0.05",
"unit_as_published": "Gb",
"context": "text-only weights; separately reported additive int8 KV cache at 32k",
"source": "https://arxiv.org/html/2607.02770v1#S2.T3",
"source_id": "quant-gemma4-report-v1",
"evidence": {
"file": "speed-evidence/gemma4-v1.html",
"sha256": "68ad60d288ff47426a7c0e30b42a4aaea0c2cde0c2912ad215db461d3f2c57bf",
"html_row_id": "S2.T3.2.2"
},
"eligible_for_automatic_variant_gb": false,
"note": "Retains exact table literals. Caption says Gb; GB interpretation is plausible from bf16 parameter sizes but not silently asserted. Not measured inference throughput, quality retention, GGUF download size, or total process RAM. Mobile representation must not be relabeled Q4_0."
},
{
"model": "gemma-4-e4b",
"model_label": "E4B",
"quant": "mobile (mixed int2/int4 weights, int8 activations)",
"bf16_weights_as_published": "9.0",
"quantized_weights_as_published": "2.3",
"int8_kv_cache_at_32k_as_published": "+0.14",
"unit_as_published": "Gb",
"context": "text-only weights; separately reported additive int8 KV cache at 32k",
"source": "https://arxiv.org/html/2607.02770v1#S2.T3",
"source_id": "quant-gemma4-report-v1",
"evidence": {
"file": "speed-evidence/gemma4-v1.html",
"sha256": "68ad60d288ff47426a7c0e30b42a4aaea0c2cde0c2912ad215db461d3f2c57bf",
"html_row_id": "S2.T3.2.3"
},
"eligible_for_automatic_variant_gb": false,
"note": "Retains exact table literals. Caption says Gb; GB interpretation is plausible from bf16 parameter sizes but not silently asserted. Not measured inference throughput, quality retention, GGUF download size, or total process RAM. Mobile representation must not be relabeled Q4_0."
},
{
"model": "gemma-4-12b",
"model_label": "12B",
"quant": "Q4_0",
"bf16_weights_as_published": "24.0",
"quantized_weights_as_published": "7.65",
"int8_kv_cache_at_32k_as_published": "+0.28",
"unit_as_published": "Gb",
"context": "text-only weights; separately reported additive int8 KV cache at 32k",
"source": "https://arxiv.org/html/2607.02770v1#S2.T3",
"source_id": "quant-gemma4-report-v1",
"evidence": {
"file": "speed-evidence/gemma4-v1.html",
"sha256": "68ad60d288ff47426a7c0e30b42a4aaea0c2cde0c2912ad215db461d3f2c57bf",
"html_row_id": "S2.T3.2.4"
},
"eligible_for_automatic_variant_gb": false,
"note": "Retains exact table literals. Caption says Gb; GB interpretation is plausible from bf16 parameter sizes but not silently asserted. Not measured inference throughput, quality retention, GGUF download size, or total process RAM. Mobile representation must not be relabeled Q4_0."
},
{
"model": "gemma-4-26b-a4b",
"model_label": "26B-A4B*",
"quant": "Q4_0",
"bf16_weights_as_published": "52.0 / 7.6",
"quantized_weights_as_published": "16.2 / 2.8",
"int8_kv_cache_at_32k_as_published": "+0.28",
"unit_as_published": "Gb",
"context": "text-only weights; separately reported additive int8 KV cache at 32k",
"source": "https://arxiv.org/html/2607.02770v1#S2.T3",
"source_id": "quant-gemma4-report-v1",
"evidence": {
"file": "speed-evidence/gemma4-v1.html",
"sha256": "68ad60d288ff47426a7c0e30b42a4aaea0c2cde0c2912ad215db461d3f2c57bf",
"html_row_id": "S2.T3.2.5"
},
"eligible_for_automatic_variant_gb": false,
"note": "Retains exact table literals. Caption says Gb; GB interpretation is plausible from bf16 parameter sizes but not silently asserted. Not measured inference throughput, quality retention, GGUF download size, or total process RAM. Mobile representation must not be relabeled Q4_0. Slash pair preserved; 2.8 is not the total resident weight footprint."
},
{
"model": "gemma-4-31b",
"model_label": "31B",
"quant": "Q4_0",
"bf16_weights_as_published": "64.0",
"quantized_weights_as_published": "19.2",
"int8_kv_cache_at_32k_as_published": "+1.10",
"unit_as_published": "Gb",
"context": "text-only weights; separately reported additive int8 KV cache at 32k",
"source": "https://arxiv.org/html/2607.02770v1#S2.T3",
"source_id": "quant-gemma4-report-v1",
"evidence": {
"file": "speed-evidence/gemma4-v1.html",
"sha256": "68ad60d288ff47426a7c0e30b42a4aaea0c2cde0c2912ad215db461d3f2c57bf",
"html_row_id": "S2.T3.2.6"
},
"eligible_for_automatic_variant_gb": false,
"note": "Retains exact table literals. Caption says Gb; GB interpretation is plausible from bf16 parameter sizes but not silently asserted. Not measured inference throughput, quality retention, GGUF download size, or total process RAM. Mobile representation must not be relabeled Q4_0."
}
],
"policy": "Reference findings only. No universal score penalty or automatic memory-size mapping inferred from these studies."
}