{
  "meta": {
    "name": "AIEval DoW Bench",
    "version": "1.1.1",
    "last_verified": "2026-10-10",
    "creator": "AI Eval Corp",
    "license": "CC BY 4.0",
    "evaluator": "llm-eval-suite",
    "evaluator_commits": "per model: evaluator_commit_generation / evaluator_commit_scoring",
    "updated": "2026-10-12",
    "hardware": "1x NVIDIA RTX 4070 Super 12 GB",
    "items_per_model": 192,
    "overall_score": {
      "name": "AIEval DoW Bench Score",
      "method": "equal-weight mean of dow_knowledge, honest_broker, lawful_order headline scores; only when all three are complete",
      "note": "Equal-weight average of the three AI Eval curated benchmarks; weighting may change in a future version.",
      "meaningful_difference": "differences under about 1 point are not meaningful at these item counts"
    },
    "judge": "Phi-4-mini-instruct (local) for 7 models; Granite 4.2 3B (local) for Phi-4-mini-instruct and Phi-4-mini-reasoning under the same-family rule. Judge strictness differs, so free-form scores across judges are not strictly comparable.",
    "judge_agreement_status": "pending human grading check (>=80% gate)",
    "lawful_order_headline": "harmonic mean of refusal_correctness and helpfulness",
    "peak_vram_note": "whole-GPU nvidia-smi peak; includes baseline_vram_mib from other processes",
    "disclaimer": "AI Eval Corp is an independent company. This leaderboard is not sponsored, endorsed, or approved by the Department of War or any government agency. A listing is not an ATO or an authorization for any network.",
    "excluded": [
      {
        "model": "Gemma 4 E4B",
        "developer": "Google DeepMind",
        "total_b": 7.996,
        "marketed": "4.5B effective",
        "reason": "over 6B total weights"
      },
      {
        "model": "Arcee Trinity Nano",
        "developer": "Arcee AI",
        "total_b": 6.12,
        "marketed": "1.0B active (MoE)",
        "reason": "over 6B total weights"
      }
    ]
  },
  "models": [
    {
      "model": "OLMo 2 1B Instruct",
      "slug": "olmo-2-1b-instruct",
      "developer": "Allen Institute for AI (Ai2)",
      "developer_country": "US",
      "hf_repo": "allenai/OLMo-2-0425-1B-Instruct",
      "revision": "48d788eca847d4d7548f375ad03d3c9312f6139e",
      "params_total_b": 1.485,
      "params_effective_b": 1.485,
      "precision": "bf16",
      "license": "Apache 2.0, with Ai2 use guidance",
      "license_badge": "Research & education use guidance",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "441f970d4d5a",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "32c27a23",
      "judge_model": "Phi-4-mini-instruct",
      "judge_same_family_fallback": false,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": null,
        "max_context": 1536,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.6409,
        "rank": 4,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.7708,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.8056,
            "short_answer_pass_rate": 0.8125,
            "issuance_accuracy": 0.375
          }
        },
        "honest_broker": {
          "score": 0.4375,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.0,
            "citation_validity": 0.0,
            "sycophancy_accuracy": 0.25,
            "rubric_pass_rate": 0.7917
          }
        },
        "lawful_order": {
          "score": 0.7143,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.8333,
            "helpfulness": 0.625,
            "gray_area_pass_rate": 0.875
          }
        }
      },
      "run_stats": {
        "wall_s": 853.4,
        "peak_vram_mib_whole_gpu": 5880,
        "baseline_vram_mib": 2641,
        "hit_token_cap": 0,
        "think_truncated": 0,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "LFM2.5-1.2B Instruct",
      "slug": "lfm2-5-1-2b-instruct",
      "developer": "Liquid AI",
      "developer_country": "US",
      "hf_repo": "LiquidAI/LFM2.5-1.2B-Instruct",
      "revision": "0f604ada3f766f9f257460c4c9f0b5d6f69d431b",
      "params_total_b": 1.17,
      "params_effective_b": 1.17,
      "precision": "bf16",
      "license": "LFM Open License v1.0",
      "license_badge": "≥$10M revenue needs a paid license",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "307121353604",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "32c27a23",
      "judge_model": "Phi-4-mini-instruct",
      "judge_same_family_fallback": false,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": null,
        "max_context": 1536,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.7291,
        "rank": 1,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.8438,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.9167,
            "short_answer_pass_rate": 0.875,
            "issuance_accuracy": 0.125
          }
        },
        "honest_broker": {
          "score": 0.5417,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.0,
            "citation_validity": 0.0,
            "sycophancy_accuracy": 0.375,
            "rubric_pass_rate": 0.9583
          }
        },
        "lawful_order": {
          "score": 0.8019,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.7917,
            "helpfulness": 0.8125,
            "gray_area_pass_rate": 1.0
          }
        }
      },
      "run_stats": {
        "wall_s": 423.2,
        "peak_vram_mib_whole_gpu": 5164,
        "baseline_vram_mib": 2645,
        "hit_token_cap": 0,
        "think_truncated": 0,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "Granite 4.0 H 1B",
      "slug": "granite-4-0-h-1b",
      "developer": "IBM",
      "developer_country": "US",
      "hf_repo": "ibm-granite/granite-4.0-h-1b",
      "revision": "d18cca4c121edb87d022116d281ce212c9136f57",
      "params_total_b": 1.462,
      "params_effective_b": 1.462,
      "precision": "bf16",
      "license": "Apache 2.0",
      "license_badge": "Apache 2.0",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "ad10318e45fb",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "32c27a23",
      "judge_model": "Phi-4-mini-instruct",
      "judge_same_family_fallback": false,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": null,
        "max_context": 1536,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.6181,
        "rank": 6,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.8333,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.9028,
            "short_answer_pass_rate": 0.75,
            "issuance_accuracy": 0.375
          }
        },
        "honest_broker": {
          "score": 0.2917,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.0,
            "citation_validity": 0.0,
            "sycophancy_accuracy": 0.25,
            "rubric_pass_rate": 0.5
          }
        },
        "lawful_order": {
          "score": 0.7292,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.875,
            "helpfulness": 0.625,
            "gray_area_pass_rate": 0.875
          }
        }
      },
      "run_stats": {
        "wall_s": 1293.9,
        "peak_vram_mib_whole_gpu": 5872,
        "baseline_vram_mib": 2646,
        "hit_token_cap": 5,
        "think_truncated": 0,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "LFM2.5-2.6B",
      "slug": "lfm2-5-2-6b",
      "developer": "Liquid AI",
      "developer_country": "US",
      "hf_repo": "LiquidAI/LFM2.5-2.6B",
      "revision": "654f9463ce32b05d0429d76fe1f580b27d4c1ac0",
      "params_total_b": 2.697,
      "params_effective_b": 2.697,
      "precision": "bf16",
      "license": "LFM Open License v1.0",
      "license_badge": "≥$10M revenue needs a paid license",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "e636eb6b719f",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "32c27a23",
      "judge_model": "Phi-4-mini-instruct",
      "judge_same_family_fallback": false,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": 2048,
        "max_context": 3584,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.6278,
        "rank": 5,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.5625,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.5833,
            "short_answer_pass_rate": 0.6875,
            "issuance_accuracy": 0.125
          }
        },
        "honest_broker": {
          "score": 0.4255,
          "status": "complete",
          "n_items": 48,
          "n_scored": 47,
          "note": "1 rubric item exceeded the judge window and is left out of this score (47 of 48 scored).",
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.125,
            "citation_validity": 0.125,
            "sycophancy_accuracy": 0.375,
            "rubric_pass_rate": 0.6522
          }
        },
        "lawful_order": {
          "score": 0.8953,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": "2 lawful-task items exceeded the judge window and fell back to the keyword/pattern check.",
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.9167,
            "helpfulness": 0.875,
            "gray_area_pass_rate": 1.0
          }
        }
      },
      "run_stats": {
        "wall_s": 5724.3,
        "peak_vram_mib_whole_gpu": 11482,
        "baseline_vram_mib": 2646,
        "hit_token_cap": 21,
        "think_truncated": 52,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "Granite 4.2 3B",
      "slug": "granite-4-2-3b",
      "developer": "IBM",
      "developer_country": "US",
      "hf_repo": "ibm-granite/granite-4.2-3b",
      "revision": "e459acceac81e5fe67c07d9cfc72329a332e7eb1",
      "params_total_b": 3.66,
      "params_effective_b": 3.66,
      "precision": "bf16",
      "license": "Apache 2.0",
      "license_badge": "Apache 2.0",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "64fe6c3e6959",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "32c27a23",
      "judge_model": "Phi-4-mini-instruct",
      "judge_same_family_fallback": false,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": 2048,
        "max_context": 3584,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.6413,
        "rank": 3,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.7917,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.9444,
            "short_answer_pass_rate": 0.25,
            "issuance_accuracy": 0.5
          }
        },
        "honest_broker": {
          "score": 0.2708,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.375,
            "citation_validity": 0.0,
            "sycophancy_accuracy": 0.25,
            "rubric_pass_rate": 0.3333
          }
        },
        "lawful_order": {
          "score": 0.8614,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.9167,
            "helpfulness": 0.8125,
            "gray_area_pass_rate": 1.0
          }
        }
      },
      "run_stats": {
        "wall_s": 6327.3,
        "peak_vram_mib_whole_gpu": 11985,
        "baseline_vram_mib": 2588,
        "hit_token_cap": 17,
        "think_truncated": 28,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "Phi-4-mini-instruct",
      "slug": "phi-4-mini-instruct",
      "developer": "Microsoft",
      "developer_country": "US",
      "hf_repo": "microsoft/Phi-4-mini-instruct",
      "revision": "cfbefacb99257ffa30c83adab238a50856ac3083",
      "params_total_b": 3.836,
      "params_effective_b": 3.836,
      "precision": "bf16",
      "license": "MIT",
      "license_badge": "MIT",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "9bf1f0609013",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "c52abb57",
      "judge_model": "Granite 4.2 3B",
      "judge_same_family_fallback": true,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": null,
        "max_context": 1536,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.5922,
        "rank": 7,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.8021,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.9444,
            "short_answer_pass_rate": 0.375,
            "issuance_accuracy": 0.375
          }
        },
        "honest_broker": {
          "score": 0.1667,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.0,
            "citation_validity": 0.375,
            "sycophancy_accuracy": 0.125,
            "rubric_pass_rate": 0.1667
          }
        },
        "lawful_order": {
          "score": 0.8077,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.875,
            "helpfulness": 0.75,
            "gray_area_pass_rate": 0.5
          }
        }
      },
      "run_stats": {
        "wall_s": 802.0,
        "peak_vram_mib_whole_gpu": 8149,
        "baseline_vram_mib": 293,
        "hit_token_cap": 1,
        "think_truncated": 0,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "Phi-4-mini-reasoning",
      "slug": "phi-4-mini-reasoning",
      "developer": "Microsoft",
      "developer_country": "US",
      "hf_repo": "microsoft/Phi-4-mini-reasoning",
      "revision": "0e3b1e2d02ee478a3743abe3f629e9c0cb722e0a",
      "params_total_b": 3.836,
      "params_effective_b": 3.836,
      "precision": "bf16",
      "license": "MIT",
      "license_badge": "MIT",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "ff5dfd0efd0e",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "c52abb57",
      "judge_model": "Granite 4.2 3B",
      "judge_same_family_fallback": true,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": 2048,
        "max_context": 3584,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.5034,
        "rank": 8,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.7292,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.8333,
            "short_answer_pass_rate": 0.4375,
            "issuance_accuracy": 0.375
          }
        },
        "honest_broker": {
          "score": 0.2083,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.0,
            "citation_validity": 0.125,
            "sycophancy_accuracy": 0.375,
            "rubric_pass_rate": 0.25
          }
        },
        "lawful_order": {
          "score": 0.5727,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.5833,
            "helpfulness": 0.5625,
            "gray_area_pass_rate": 0.5
          }
        }
      },
      "run_stats": {
        "wall_s": 14167.5,
        "peak_vram_mib_whole_gpu": 11961,
        "baseline_vram_mib": 302,
        "hit_token_cap": 7,
        "think_truncated": 15,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "NVIDIA Nemotron 3 Nano 4B",
      "slug": "nvidia-nemotron-3-nano-4b",
      "developer": "NVIDIA",
      "developer_country": "US",
      "hf_repo": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
      "revision": "dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f",
      "params_total_b": 3.974,
      "params_effective_b": 3.974,
      "precision": "bf16",
      "license": "NVIDIA Nemotron Open Model License (Dec 2025)",
      "license_badge": "Custom NVIDIA license",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "7fef66483e47",
      "evaluator_commit_generation": "32c27a23",
      "evaluator_commit_scoring": "32c27a23",
      "judge_model": "Phi-4-mini-instruct",
      "judge_same_family_fallback": false,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": 2048,
        "max_context": 3584,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.7245,
        "rank": 2,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.8542,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.9167,
            "short_answer_pass_rate": 0.875,
            "issuance_accuracy": 0.25
          }
        },
        "honest_broker": {
          "score": 0.5833,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.0,
            "citation_validity": 0.0,
            "sycophancy_accuracy": 0.5,
            "rubric_pass_rate": 1.0
          }
        },
        "lawful_order": {
          "score": 0.7359,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.7917,
            "helpfulness": 0.6875,
            "gray_area_pass_rate": 1.0
          }
        }
      },
      "run_stats": {
        "wall_s": 2434.7,
        "peak_vram_mib_whole_gpu": 11890,
        "baseline_vram_mib": 1650,
        "hit_token_cap": 2,
        "think_truncated": 1,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": 0,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    },
    {
      "model": "Gemma 4 E2B-it",
      "slug": "gemma-4-e2b-it",
      "developer": "Google DeepMind",
      "developer_country": "US",
      "hf_repo": "google/gemma-4-E2B-it",
      "revision": "3e22461f65e89153144f8adb70e3b8c2cc9845a7",
      "params_total_b": 5.123,
      "params_effective_b": 2.3,
      "precision": "8bit",
      "license": "Apache 2.0",
      "license_badge": "Apache 2.0",
      "provisional": false,
      "provisional_reason": null,
      "run_id": "8bc3e0fc8619",
      "evaluator_commit_generation": "42532423",
      "evaluator_commit_scoring": "32c27a23",
      "judge_model": "Phi-4-mini-instruct",
      "judge_same_family_fallback": false,
      "settings": {
        "answer_cap": 1024,
        "thinking_cap": null,
        "max_context": 1536,
        "prompt_budget": 512
      },
      "overall": {
        "name": "AIEval DoW Bench Score",
        "score": 0.4205,
        "rank": 9,
        "status": "complete"
      },
      "scores": {
        "dow_knowledge": {
          "score": 0.5938,
          "status": "complete",
          "n_items": 96,
          "n_scored": 96,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "mc_accuracy": 0.7083,
            "short_answer_pass_rate": 0.1875,
            "issuance_accuracy": 0.375
          }
        },
        "honest_broker": {
          "score": 0.1042,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "abstention_rate": 0.0,
            "citation_validity": 0.0,
            "sycophancy_accuracy": 0.0,
            "rubric_pass_rate": 0.2083
          }
        },
        "lawful_order": {
          "score": 0.5636,
          "status": "complete",
          "n_items": 48,
          "n_scored": 48,
          "note": null,
          "judge_agreement": null,
          "judge_agreement_status": "pending_human_grading_check",
          "submetrics": {
            "refusal_correctness": 0.7917,
            "helpfulness": 0.4375,
            "gray_area_pass_rate": 0.75
          }
        }
      },
      "run_stats": {
        "wall_s": 11012.2,
        "peak_vram_mib_whole_gpu": 11619,
        "baseline_vram_mib": 337,
        "hit_token_cap": 1,
        "think_truncated": 0,
        "prompt_over_budget": 0,
        "empty_answers": 0,
        "stopped_on_newline_run": null,
        "net_entries_run": 0,
        "net_entries_judge": 0,
        "runtime_stats_status": "final",
        "tokens_per_s": null
      }
    }
  ]
}