{
  "prepared": "2026-10-28",
  "status": "review draft; archived outcomes, not a controlled pass@1 leaderboard",
  "tasks_per_model": 3147,
  "rows": [
    {
      "model": "afm3",
      "name": "Apple AFM (system default; variant unverified)",
      "suite": "humaneval",
      "label": "HumanEval",
      "passed": 103,
      "total": 164,
      "saved_success_pct": 62.8,
      "retry_flags": {},
      "report": "logs/eval_report_afm3_humaneval_full.json",
      "sha256": "d4f2355e658e988b20f45ec00cc0f61ac083b8cee257ab2f135ecab8f6398285",
      "timestamp": 1789857265
    },
    {
      "model": "afm3",
      "name": "Apple AFM (system default; variant unverified)",
      "suite": "tooluse",
      "label": "Telos tool-use",
      "passed": 22,
      "total": 30,
      "saved_success_pct": 73.33,
      "retry_flags": {},
      "report": "logs/eval_report_afm3_tooluse_full.json",
      "sha256": "674c5222e6456d64694f20ff1398764254abda6018d2c9246f14e95e241368c2",
      "timestamp": 1789834882
    },
    {
      "model": "afm3",
      "name": "Apple AFM (system default; variant unverified)",
      "suite": "cyber",
      "label": "Telos cybersecurity",
      "passed": 6,
      "total": 50,
      "saved_success_pct": 12.0,
      "retry_flags": {},
      "report": "logs/eval_report_afm3_cyber_full.json",
      "sha256": "0f138d363bb38983ca4ac4ae19f8fc3308e3577e8224e7969432e2c40823dab1",
      "timestamp": 1789896505
    },
    {
      "model": "afm3",
      "name": "Apple AFM (system default; variant unverified)",
      "suite": "gpqa_diamond",
      "label": "GPQA Diamond",
      "passed": 78,
      "total": 198,
      "saved_success_pct": 39.39,
      "retry_flags": {
        "recovered_with_4096_tokens": 28
      },
      "report": "logs/eval_report_afm3_gpqa_diamond_full.json",
      "sha256": "7dd661f3303f15b7a0216302a4e0b72072b2d64be9d95c63eb659628531c1fc4",
      "timestamp": 1789842458
    },
    {
      "model": "afm3",
      "name": "Apple AFM (system default; variant unverified)",
      "suite": "mmlu_science",
      "label": "MMLU Science selection",
      "passed": 581,
      "total": 833,
      "saved_success_pct": 69.75,
      "retry_flags": {},
      "report": "logs/eval_report_afm3_mmlu_science_full.json",
      "sha256": "cfb7719043729e3cd096fd65c51a6cc88e4c47a7a2cc283029e8be8286ba3aec",
      "timestamp": 1789836507
    },
    {
      "model": "afm3",
      "name": "Apple AFM (system default; variant unverified)",
      "suite": "competition_math",
      "label": "Competition MATH selection",
      "passed": 364,
      "total": 700,
      "saved_success_pct": 52.0,
      "retry_flags": {
        "recovered_with_4096_tokens": 49
      },
      "report": "logs/eval_report_afm3_competition_math_full.json",
      "sha256": "69edc62b61414b8d3bf37e51bf54321f01acebc7f61e856901dc700af7274bc3",
      "timestamp": 1789845413
    },
    {
      "model": "afm3",
      "name": "Apple AFM (system default; variant unverified)",
      "suite": "arc",
      "label": "ARC-Challenge",
      "passed": 1000,
      "total": 1172,
      "saved_success_pct": 85.32,
      "retry_flags": {},
      "report": "logs/eval_report_afm3_arc_full.json",
      "sha256": "dc71e1ccfd20149698bdddd103214ae75e5d8b1182948ded890f0c223521a0c6",
      "timestamp": 1789852846
    },
    {
      "model": "ternary_bonsai_4b",
      "name": "Ternary Bonsai 4B (2-bit MLX)",
      "suite": "humaneval",
      "label": "HumanEval",
      "passed": 118,
      "total": 164,
      "saved_success_pct": 71.95,
      "retry_flags": {},
      "report": "logs/ternary_bonsai_4b_mac_20261018/humaneval_report.json",
      "sha256": "6ec5593f0f4e93f7efa3e157654c055dc2e9a97cd36152a405139ae408c76ad2",
      "timestamp": 1792376533
    },
    {
      "model": "ternary_bonsai_4b",
      "name": "Ternary Bonsai 4B (2-bit MLX)",
      "suite": "tooluse",
      "label": "Telos tool-use",
      "passed": 19,
      "total": 30,
      "saved_success_pct": 63.33,
      "retry_flags": {},
      "report": "logs/ternary_bonsai_4b_mac_20261018/tooluse_report.json",
      "sha256": "c543b29aa788275aee82146b9bd31e72778df70133a875c4352acf3bb7dad2e7",
      "timestamp": 1792377956
    },
    {
      "model": "ternary_bonsai_4b",
      "name": "Ternary Bonsai 4B (2-bit MLX)",
      "suite": "cyber",
      "label": "Telos cybersecurity",
      "passed": 8,
      "total": 50,
      "saved_success_pct": 16.0,
      "retry_flags": {},
      "report": "logs/ternary_bonsai_4b_mac_20261018/cyber_report.json",
      "sha256": "508d3e7d4661862623d7213031b7b1365e49d4f39234072cc5c2c9d0506bfb14",
      "timestamp": 1792377964
    },
    {
      "model": "ternary_bonsai_4b",
      "name": "Ternary Bonsai 4B (2-bit MLX)",
      "suite": "gpqa_diamond",
      "label": "GPQA Diamond",
      "passed": 53,
      "total": 198,
      "saved_success_pct": 26.77,
      "retry_flags": {},
      "report": "logs/ternary_bonsai_4b_mac_20261018/gpqa_diamond_report.json",
      "sha256": "7f0ebe1ab13f897b76313937c579b179c37198039d1a916fcca3922893726443",
      "timestamp": 1792340516
    },
    {
      "model": "ternary_bonsai_4b",
      "name": "Ternary Bonsai 4B (2-bit MLX)",
      "suite": "mmlu_science",
      "label": "MMLU Science selection",
      "passed": 521,
      "total": 833,
      "saved_success_pct": 62.55,
      "retry_flags": {},
      "report": "logs/ternary_bonsai_4b_mac_20261018/mmlu_science_report.json",
      "sha256": "4893ede8e29f1b1ea8dcb662857f6c93f81c0f63f2c90b2ff0f97f7644c32159",
      "timestamp": 1792323731
    },
    {
      "model": "ternary_bonsai_4b",
      "name": "Ternary Bonsai 4B (2-bit MLX)",
      "suite": "competition_math",
      "label": "Competition MATH selection",
      "passed": 397,
      "total": 700,
      "saved_success_pct": 56.71,
      "retry_flags": {},
      "report": "logs/ternary_bonsai_4b_mac_20261018/competition_math_report.json",
      "sha256": "7a17297bfed181b65d3dd798f0c4724f8e8d9cc853d40da9fbf3e39ff23978b3",
      "timestamp": 1792353619
    },
    {
      "model": "ternary_bonsai_4b",
      "name": "Ternary Bonsai 4B (2-bit MLX)",
      "suite": "arc",
      "label": "ARC-Challenge",
      "passed": 640,
      "total": 1172,
      "saved_success_pct": 54.61,
      "retry_flags": {},
      "report": "logs/ternary_bonsai_4b_mac_20261018/arc_report.json",
      "sha256": "cbf1df4a9d64a60814cda6309509440df155dd97bf535a2153fb0c0cbd0c5346",
      "timestamp": 1792319010
    },
    {
      "model": "granite_mlx",
      "name": "IBM Granite 4.2 3B (saved run label)",
      "suite": "humaneval",
      "label": "HumanEval",
      "passed": 99,
      "total": 164,
      "saved_success_pct": 60.37,
      "retry_flags": {},
      "report": "logs/eval_report_granite_mlx_humaneval_full.json",
      "sha256": "3a54d2fa894bbfe1c5d0a10ab504576e75a450a2f6d55eba6870ac5f313ed6e7",
      "timestamp": 1789870977
    },
    {
      "model": "granite_mlx",
      "name": "IBM Granite 4.2 3B (saved run label)",
      "suite": "tooluse",
      "label": "Telos tool-use",
      "passed": 21,
      "total": 30,
      "saved_success_pct": 70.0,
      "retry_flags": {},
      "report": "logs/eval_report_granite_mlx_tooluse_full.json",
      "sha256": "d1a8c39c957e0d0876e2c700cb91da52b1a7a90f6a531c3fd5e0ccaf01c65e49",
      "timestamp": 1789875062
    },
    {
      "model": "granite_mlx",
      "name": "IBM Granite 4.2 3B (saved run label)",
      "suite": "cyber",
      "label": "Telos cybersecurity",
      "passed": 11,
      "total": 50,
      "saved_success_pct": 22.0,
      "retry_flags": {},
      "report": "logs/eval_report_granite_mlx_cyber_full.json",
      "sha256": "066695ea102983d47dcc02fc8bc8916d85b857eb26ba61b3d14ca2978e4f1d6a",
      "timestamp": 1789875178
    },
    {
      "model": "granite_mlx",
      "name": "IBM Granite 4.2 3B (saved run label)",
      "suite": "gpqa_diamond",
      "label": "GPQA Diamond",
      "passed": 57,
      "total": 198,
      "saved_success_pct": 28.79,
      "retry_flags": {
        "recovered_with_4096_tokens": 35
      },
      "report": "logs/eval_report_granite_mlx_gpqa_diamond_full.json",
      "sha256": "3f6e545699723e2753a100af19483328709d9062c74d884fce9c38a19385d91c",
      "timestamp": 1789896817
    },
    {
      "model": "granite_mlx",
      "name": "IBM Granite 4.2 3B (saved run label)",
      "suite": "mmlu_science",
      "label": "MMLU Science selection",
      "passed": 515,
      "total": 833,
      "saved_success_pct": 61.82,
      "retry_flags": {
        "repaired_16000_tokens": 16
      },
      "report": "logs/eval_report_granite_mlx_mmlu_science_full.json",
      "sha256": "423c34b59c5bb06928a21fec3d117a4115072ccad07aca9c3bcaee24dca09b34",
      "timestamp": 1789900022
    },
    {
      "model": "granite_mlx",
      "name": "IBM Granite 4.2 3B (saved run label)",
      "suite": "competition_math",
      "label": "Competition MATH selection",
      "passed": 453,
      "total": 700,
      "saved_success_pct": 64.71,
      "retry_flags": {
        "recovered_with_4096_tokens": 204
      },
      "report": "logs/eval_report_granite_mlx_competition_math_full.json",
      "sha256": "f4caccd117da2d02ac770ab36553d221776f8d875ffde28bdfd7d2a89654533b",
      "timestamp": 1789885573
    },
    {
      "model": "granite_mlx",
      "name": "IBM Granite 4.2 3B (saved run label)",
      "suite": "arc",
      "label": "ARC-Challenge",
      "passed": 1015,
      "total": 1172,
      "saved_success_pct": 86.6,
      "retry_flags": {
        "repaired_16000_tokens": 572
      },
      "report": "logs/eval_report_granite_mlx_arc_full.json",
      "sha256": "02c260e4d6694d3e234be9130f8f30885fb5446708819e554b59d2292bfba566",
      "timestamp": 1789881505
    },
    {
      "model": "lfm8b",
      "name": "LiquidAI LFM 2.5 8B A1B (saved run label)",
      "suite": "humaneval",
      "label": "HumanEval",
      "passed": 75,
      "total": 164,
      "saved_success_pct": 45.73,
      "retry_flags": {},
      "report": "logs/eval_report_lfm8b_humaneval_full.json",
      "sha256": "6d90478839d515483391ebbd198bf1832d853758f69e726ee01e89bda958b01c",
      "timestamp": 1789915634
    },
    {
      "model": "lfm8b",
      "name": "LiquidAI LFM 2.5 8B A1B (saved run label)",
      "suite": "tooluse",
      "label": "Telos tool-use",
      "passed": 19,
      "total": 30,
      "saved_success_pct": 63.33,
      "retry_flags": {},
      "report": "logs/eval_report_lfm8b_tooluse_full.json",
      "sha256": "5152eb231db643a4a13edda57f9bbc37cc4365696806d58717b1a1382b28aa9c",
      "timestamp": 1789918849
    },
    {
      "model": "lfm8b",
      "name": "LiquidAI LFM 2.5 8B A1B (saved run label)",
      "suite": "cyber",
      "label": "Telos cybersecurity",
      "passed": 12,
      "total": 50,
      "saved_success_pct": 24.0,
      "retry_flags": {},
      "report": "logs/eval_report_lfm8b_cyber_full.json",
      "sha256": "8db7a507c5a8c267ebfd6dca493220de1f9a6497c148502920c75d0d922985c8",
      "timestamp": 1789918896
    },
    {
      "model": "lfm8b",
      "name": "LiquidAI LFM 2.5 8B A1B (saved run label)",
      "suite": "gpqa_diamond",
      "label": "GPQA Diamond",
      "passed": 41,
      "total": 198,
      "saved_success_pct": 20.71,
      "retry_flags": {},
      "report": "logs/eval_report_lfm8b_gpqa_diamond_full.json",
      "sha256": "a840c250b513cc6e2b7792ec1bb0c7bfc90d750077f2bd101387d38376fb594a",
      "timestamp": 1789919328
    },
    {
      "model": "lfm8b",
      "name": "LiquidAI LFM 2.5 8B A1B (saved run label)",
      "suite": "mmlu_science",
      "label": "MMLU Science selection",
      "passed": 609,
      "total": 833,
      "saved_success_pct": 73.11,
      "retry_flags": {},
      "report": "logs/eval_report_lfm8b_mmlu_science_full.json",
      "sha256": "924866dd3067421133128336cc566141555530885e674bd9032eb7716447d382",
      "timestamp": 1789920919
    },
    {
      "model": "lfm8b",
      "name": "LiquidAI LFM 2.5 8B A1B (saved run label)",
      "suite": "competition_math",
      "label": "Competition MATH selection",
      "passed": 305,
      "total": 700,
      "saved_success_pct": 43.57,
      "retry_flags": {},
      "report": "logs/eval_report_lfm8b_competition_math_full.json",
      "sha256": "2aaf3bec80b13d769ef50d2167cd6230b5ffcd460b9870bf0dc06d1c05565c3f",
      "timestamp": 1789929373
    },
    {
      "model": "lfm8b",
      "name": "LiquidAI LFM 2.5 8B A1B (saved run label)",
      "suite": "arc",
      "label": "ARC-Challenge",
      "passed": 698,
      "total": 1172,
      "saved_success_pct": 59.56,
      "retry_flags": {},
      "report": "logs/eval_report_lfm8b_arc_full.json",
      "sha256": "d748e4901d156d1ff58738b0b347ad78c9ccd3534ae092f60429c10dbe0a01a1",
      "timestamp": 1789933601
    },
    {
      "model": "gemma4_e4b",
      "name": "Google Gemma 4 E4B (4-bit MLX)",
      "suite": "humaneval",
      "label": "HumanEval",
      "passed": 135,
      "total": 164,
      "saved_success_pct": 82.32,
      "retry_flags": {},
      "report": "logs/gemma4_mac_overnight_20261017/humaneval_report.json",
      "sha256": "5225646dbec61b86e5b9eaba5ac9129f290fdebb3652061492151bb2b2985695",
      "timestamp": 1792313613
    },
    {
      "model": "gemma4_e4b",
      "name": "Google Gemma 4 E4B (4-bit MLX)",
      "suite": "tooluse",
      "label": "Telos tool-use",
      "passed": 24,
      "total": 30,
      "saved_success_pct": 80.0,
      "retry_flags": {},
      "report": "logs/gemma4_mac_overnight_20261017/tooluse_report.json",
      "sha256": "3fae1a9aed18cf93ea2a66defed542ef7b5dfcd299e46249cbae38ade5ad7dd2",
      "timestamp": 1792315849
    },
    {
      "model": "gemma4_e4b",
      "name": "Google Gemma 4 E4B (4-bit MLX)",
      "suite": "cyber",
      "label": "Telos cybersecurity",
      "passed": 28,
      "total": 50,
      "saved_success_pct": 56.0,
      "retry_flags": {},
      "report": "logs/gemma4_mac_overnight_20261017/cyber_report.json",
      "sha256": "feeca98b2c039330ae3740c65e857d331a24763eeb474f785af861b98a9d5877",
      "timestamp": 1792315869
    },
    {
      "model": "gemma4_e4b",
      "name": "Google Gemma 4 E4B (4-bit MLX)",
      "suite": "gpqa_diamond",
      "label": "GPQA Diamond",
      "passed": 87,
      "total": 198,
      "saved_success_pct": 43.94,
      "retry_flags": {},
      "report": "logs/gemma4_mac_overnight_20261017/gpqa_diamond_report.json",
      "sha256": "8de80f62a6505ac10319bcc03b77977f888666eb0b2d255f2204f88a187a453c",
      "timestamp": 1792284977
    },
    {
      "model": "gemma4_e4b",
      "name": "Google Gemma 4 E4B (4-bit MLX)",
      "suite": "mmlu_science",
      "label": "MMLU Science selection",
      "passed": 488,
      "total": 833,
      "saved_success_pct": 58.58,
      "retry_flags": {},
      "report": "logs/gemma4_mac_overnight_20261017/mmlu_science_report.json",
      "sha256": "9662874a435096c4ba3913926376b083de9f4d498090f592848d82c647a48c50",
      "timestamp": 1792274821
    },
    {
      "model": "gemma4_e4b",
      "name": "Google Gemma 4 E4B (4-bit MLX)",
      "suite": "competition_math",
      "label": "Competition MATH selection",
      "passed": 470,
      "total": 700,
      "saved_success_pct": 67.14,
      "retry_flags": {},
      "report": "logs/gemma4_mac_overnight_20261017/competition_math_report.json",
      "sha256": "0b36defb4f601af805758bdf7fd6e151a5b4c82a873c5854a7c995b98d92bce8",
      "timestamp": 1792292435
    },
    {
      "model": "gemma4_e4b",
      "name": "Google Gemma 4 E4B (4-bit MLX)",
      "suite": "arc",
      "label": "ARC-Challenge",
      "passed": 782,
      "total": 1172,
      "saved_success_pct": 66.72,
      "retry_flags": {},
      "report": "logs/gemma4_mac_overnight_20261017/arc_report.json",
      "sha256": "0979fefe275dc7ddee35f57d3530c01877ad5022e6ac084bc2bfb50003cfd69c",
      "timestamp": 1792265092
    }
  ],
  "summary_sources": [
    {
      "path": "logs/eval_report_afm3_master_summary.json",
      "sha256": "90e9fb7284a40b2a85d5dcef73ba5b415c249de95478c29aa1cb7c06342d6d92"
    },
    {
      "path": "logs/eval_report_ternary_bonsai_4b_master_summary.json",
      "sha256": "eaaefb9047b30d7592b8a474c3ae73999ce88c1bbbce3e4fce134234b4802c0b"
    },
    {
      "path": "logs/eval_report_granite_mlx_master_summary.json",
      "sha256": "fb43ab6cf19ae784637f6308e7b70f7729cb3a7103dfec924b8930ccb1a55b53"
    },
    {
      "path": "logs/eval_report_lfm8b_master_summary.json",
      "sha256": "f159e88c1e01b974accaa3689df4382daf8296a22c5c25ca60b54c85e9f5d7d3"
    },
    {
      "path": "logs/eval_report_gemma4_e4b_master_summary.json",
      "sha256": "daf7485d0f76478ebae5d5ae229e64349fefe15f4d9e43f2e4a66b1e300db77f"
    }
  ],
  "summary_discrepancies": [
    {
      "model": "ternary_bonsai_4b",
      "suite": "humaneval",
      "field": "pass_at_1_pct",
      "summary": 73.78,
      "detailed_report": 71.95
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "humaneval",
      "field": "ast_validity_pct",
      "summary": 100.0,
      "detailed_report": 89.02
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "humaneval",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 121,
        "FAILED_ASSERTION": 43
      },
      "detailed_report": {
        "PASSED": 118,
        "FAILED_ASSERTION": 22,
        "SYNTAX_ERROR": 18,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 6,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "cyber",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 8,
        "FAILED_ASSERTION": 42
      },
      "detailed_report": {
        "PASSED": 8,
        "FAILED_ASSERTION": 42,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "gpqa_diamond",
      "field": "pass_at_1_pct",
      "summary": 32.32,
      "detailed_report": 26.77
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "gpqa_diamond",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 64,
        "FAILED_ASSERTION": 134
      },
      "detailed_report": {
        "PASSED": 53,
        "FAILED_ASSERTION": 145,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "mmlu_science",
      "field": "pass_at_1_pct",
      "summary": 63.75,
      "detailed_report": 62.55
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "mmlu_science",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 531,
        "FAILED_ASSERTION": 302
      },
      "detailed_report": {
        "PASSED": 521,
        "FAILED_ASSERTION": 312,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "competition_math",
      "field": "pass_at_1_pct",
      "summary": 58.14,
      "detailed_report": 56.71
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "competition_math",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 407,
        "FAILED_ASSERTION": 293
      },
      "detailed_report": {
        "PASSED": 397,
        "FAILED_ASSERTION": 303,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "arc",
      "field": "pass_at_1_pct",
      "summary": 55.38,
      "detailed_report": 54.61
    },
    {
      "model": "ternary_bonsai_4b",
      "suite": "arc",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 649,
        "FAILED_ASSERTION": 523
      },
      "detailed_report": {
        "PASSED": 640,
        "FAILED_ASSERTION": 532,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "gemma4_e4b",
      "suite": "humaneval",
      "field": "ast_validity_pct",
      "summary": 100.0,
      "detailed_report": 96.34
    },
    {
      "model": "gemma4_e4b",
      "suite": "humaneval",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 135,
        "FAILED_ASSERTION": 29
      },
      "detailed_report": {
        "PASSED": 135,
        "FAILED_ASSERTION": 20,
        "SYNTAX_ERROR": 6,
        "TIMEOUT": 1,
        "RUNTIME_EXCEPTION": 2,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "gemma4_e4b",
      "suite": "cyber",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 28,
        "FAILED_ASSERTION": 22
      },
      "detailed_report": {
        "PASSED": 28,
        "FAILED_ASSERTION": 22,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "gemma4_e4b",
      "suite": "gpqa_diamond",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 87,
        "FAILED_ASSERTION": 111
      },
      "detailed_report": {
        "PASSED": 87,
        "FAILED_ASSERTION": 111,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "gemma4_e4b",
      "suite": "mmlu_science",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 488,
        "FAILED_ASSERTION": 345
      },
      "detailed_report": {
        "PASSED": 488,
        "FAILED_ASSERTION": 345,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "gemma4_e4b",
      "suite": "competition_math",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 470,
        "FAILED_ASSERTION": 230
      },
      "detailed_report": {
        "PASSED": 470,
        "FAILED_ASSERTION": 230,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    },
    {
      "model": "gemma4_e4b",
      "suite": "arc",
      "field": "execution_outcomes",
      "summary": {
        "PASSED": 782,
        "FAILED_ASSERTION": 390
      },
      "detailed_report": {
        "PASSED": 782,
        "FAILED_ASSERTION": 390,
        "SYNTAX_ERROR": 0,
        "TIMEOUT": 0,
        "RUNTIME_EXCEPTION": 0,
        "MEMORY_EXCEEDED": 0
      }
    }
  ]
}
