{
 "updated_at": "2026-08-15",
 "what_this_is": "teai.io の同一エンドポイントに全モデルをぶら下げて、コスト・速度・正確性を同じ条件で測った実測値。",
 "method": {
  "temperature": 0,
  "grading": "全て機械採点。正解も設問もコードで生成する(手で書いた正解は書いた人の知識で頭打ちになる)。",
  "cost": "実トークン × カタログ単価。体感でなく実測。コストは各評価セットの全問を1回通したときの合計。",
  "repeats": "同じ設問でも実行ごとに振れるため複数回まわして平均する。"
 },
 "evals": [
  {
   "id": "jp-business",
   "name": "日本の実務計算",
   "description": "税・端数処理・営業日・法定値など。正解はコードで生成。",
   "items": 182,
   "repeats": 3,
   "calls": 7644,
   "saturated_kinds": [],
   "saturated_note": ""
  },
  {
   "id": "general",
   "name": "指示追従・多段推論・コード・長文",
   "description": "形式の遵守、事実を2〜5段たどる、実行して通るコード、長文からの抽出。",
   "items": 119,
   "repeats": 2,
   "calls": 3332,
   "saturated_kinds": [
    "長文抽出"
   ],
   "saturated_note": "全モデルが満点だった種別。難易度が低すぎて差を測れていない。「差が無い」という証拠にはならない。"
  },
  {
   "id": "hard",
   "name": "難しい版",
   "description": "約1万トークンの文書に偽物を混ぜた照合・散在数値の集計・境界条件で落ちるコード・6〜8段+囮の推論。",
   "items": 46,
   "repeats": 2,
   "calls": 1288,
   "saturated_kinds": [],
   "saturated_note": ""
  },
  {
   "id": "jp-business-hard",
   "name": "日本の実務計算（難問）",
   "description": "源泉徴収の100万円境界・税込逆算の二重端数・送料のちょうど境界・祝日をまたぐ営業日・比例付与・深い法定値。既存セットで Flash が弱かった種別(記憶・表参照)の難問26問。",
   "items": 26,
   "repeats": 0,
   "calls": 0,
   "saturated_kinds": [],
   "saturated_note": ""
  }
 ],
 "models": {
  "claude-fable-5": {
   "id": "claude-fable-5",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 98.5,
     "cost_usd": 0.7565,
     "median_s": 3.84,
     "p90_s": 5.86,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 95.6,
      "計算": 98.9,
      "計算(表参照)": 100.0,
      "記憶(法定値)": 98.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 95.8,
     "cost_usd": 2.2932,
     "median_s": 3.8,
     "p90_s": 5.58,
     "errors": 0,
     "by_kind": {
      "コード生成": 88.9,
      "多段推論": 100.0,
      "指示追従": 90.9,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 97.8,
     "cost_usd": 3.4877,
     "median_s": 4.99,
     "p90_s": 6.43,
     "errors": 0,
     "by_kind": {
      "コード実装": 87.5,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "claude-sonnet-5": {
   "id": "claude-sonnet-5",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 97.6,
     "cost_usd": 0.2921,
     "median_s": 1.69,
     "p90_s": 4.12,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 97.2,
      "計算(表参照)": 99.2,
      "記憶(法定値)": 92.2
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 99.2,
     "cost_usd": 0.698,
     "median_s": 1.78,
     "p90_s": 3.0,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 97.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 100.0,
     "cost_usd": 1.0436,
     "median_s": 2.33,
     "p90_s": 3.71,
     "errors": 0,
     "by_kind": {
      "コード実装": 100.0,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "claude-haiku-4-5-20251001": {
   "id": "claude-haiku-4-5-20251001",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 81.5,
     "cost_usd": 0.0515,
     "median_s": 1.06,
     "p90_s": 2.01,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 96.7,
      "計算": 78.4,
      "計算(表参照)": 84.6,
      "記憶(法定値)": 64.7
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 92.4,
     "cost_usd": 0.232,
     "median_s": 1.06,
     "p90_s": 2.3,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 72.7,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 80.4,
     "cost_usd": 0.3443,
     "median_s": 1.39,
     "p90_s": 2.25,
     "errors": 0,
     "by_kind": {
      "コード実装": 87.5,
      "多段推論(難)": 100.0,
      "長文照合": 83.3,
      "長文集計": 25.0
     }
    }
   }
  },
  "openai/gpt-5.6-sol": {
   "id": "openai/gpt-5.6-sol",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 99.5,
     "cost_usd": 0.2775,
     "median_s": 1.4,
     "p90_s": 2.76,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 98.9,
      "計算(表参照)": 100.0,
      "記憶(法定値)": 100.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 100.0,
     "cost_usd": 1.0093,
     "median_s": 1.31,
     "p90_s": 2.82,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 100.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 98.9,
     "cost_usd": 1.5758,
     "median_s": 1.83,
     "p90_s": 3.38,
     "errors": 0,
     "by_kind": {
      "コード実装": 93.8,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "x-ai/grok-4.5": {
   "id": "x-ai/grok-4.5",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 94.7,
     "cost_usd": 0.3774,
     "median_s": 4.95,
     "p90_s": 11.73,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 93.3,
      "計算(表参照)": 91.9,
      "記憶(法定値)": 100.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 99.2,
     "cost_usd": 0.5086,
     "median_s": 4.45,
     "p90_s": 10.99,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 97.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 98.9,
     "cost_usd": 0.5835,
     "median_s": 4.2,
     "p90_s": 6.86,
     "errors": 0,
     "by_kind": {
      "コード実装": 93.8,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "moonshotai/kimi-k3": {
   "id": "moonshotai/kimi-k3",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 99.6,
     "cost_usd": 0.3308,
     "median_s": 1.63,
     "p90_s": 3.41,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 99.3,
      "計算(表参照)": 100.0,
      "記憶(法定値)": 100.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 99.6,
     "cost_usd": 0.7471,
     "median_s": 3.44,
     "p90_s": 10.45,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 98.5,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 95.7,
     "cost_usd": 1.0527,
     "median_s": 5.69,
     "p90_s": 9.7,
     "errors": 0,
     "by_kind": {
      "コード実装": 75.0,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "deepseek/deepseek-v4-pro": {
   "id": "deepseek/deepseek-v4-pro",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 89.7,
     "cost_usd": 0.0379,
     "median_s": 5.03,
     "p90_s": 11.54,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 89.0,
      "計算(表参照)": 85.4,
      "記憶(法定値)": 86.3
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 98.3,
     "cost_usd": 0.082,
     "median_s": 6.89,
     "p90_s": 15.71,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 98.4,
      "指示追従": 95.5,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 98.9,
     "cost_usd": 0.1188,
     "median_s": 7.05,
     "p90_s": 15.23,
     "errors": 0,
     "by_kind": {
      "コード実装": 93.8,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "deepseek/deepseek-v4-flash-0731": {
   "id": "deepseek/deepseek-v4-flash-0731",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 84.4,
     "cost_usd": 0.0022,
     "median_s": 0.77,
     "p90_s": 1.54,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 94.4,
      "計算": 90.8,
      "計算(表参照)": 70.7,
      "記憶(法定値)": 64.7
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 97.1,
     "cost_usd": 0.0145,
     "median_s": 0.85,
     "p90_s": 2.51,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 92.2,
      "指示追従": 97.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 93.5,
     "cost_usd": 0.0227,
     "median_s": 1.12,
     "p90_s": 8.42,
     "errors": 0,
     "by_kind": {
      "コード実装": 100.0,
      "多段推論(難)": 94.4,
      "長文照合": 100.0,
      "長文集計": 75.0
     }
    }
   }
  },
  "z-ai/glm-5.2": {
   "id": "z-ai/glm-5.2",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 97.8,
     "cost_usd": 0.1293,
     "median_s": 1.89,
     "p90_s": 4.11,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 96.5,
      "計算(表参照)": 100.0,
      "記憶(法定値)": 96.1
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 99.2,
     "cost_usd": 0.2353,
     "median_s": 1.66,
     "p90_s": 6.47,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 97.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 100.0,
     "cost_usd": 0.3005,
     "median_s": 1.9,
     "p90_s": 6.73,
     "errors": 0,
     "by_kind": {
      "コード実装": 100.0,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "qwen/qwen3.5-397b-a17b": {
   "id": "qwen/qwen3.5-397b-a17b",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 85.2,
     "cost_usd": 0.3479,
     "median_s": 11.47,
     "p90_s": 95.51,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 98.9,
      "計算": 83.3,
      "計算(表参照)": 85.4,
      "記憶(法定値)": 70.6
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 98.7,
     "cost_usd": 0.1415,
     "median_s": 10.96,
     "p90_s": 12.86,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 98.4,
      "指示追従": 97.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 95.7,
     "cost_usd": 0.1428,
     "median_s": 12.22,
     "p90_s": 17.02,
     "errors": 0,
     "by_kind": {
      "コード実装": 93.8,
      "多段推論(難)": 100.0,
      "長文照合": 87.5,
      "長文集計": 100.0
     }
    }
   }
  },
  "google/gemini-3.1-pro-preview": {
   "id": "google/gemini-3.1-pro-preview",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 99.5,
     "cost_usd": 0.5678,
     "median_s": 3.44,
     "p90_s": 6.06,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 98.9,
      "計算(表参照)": 100.0,
      "記憶(法定値)": 100.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 100.0,
     "cost_usd": 0.5686,
     "median_s": 3.45,
     "p90_s": 5.5,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 100.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 100.0,
     "cost_usd": 0.6985,
     "median_s": 5.08,
     "p90_s": 6.93,
     "errors": 0,
     "by_kind": {
      "コード実装": 100.0,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "sakana/fugu-ultra": {
   "id": "sakana/fugu-ultra",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 90.5,
     "cost_usd": 4.6914,
     "median_s": 13.12,
     "p90_s": 31.73,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 85.8,
      "計算(表参照)": 95.9,
      "記憶(法定値)": 86.3
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 97.1,
     "cost_usd": 4.0579,
     "median_s": 15.57,
     "p90_s": 28.9,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 95.3,
      "指示追従": 93.9,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 95.7,
     "cost_usd": 2.9552,
     "median_s": 16.76,
     "p90_s": 26.52,
     "errors": 0,
     "by_kind": {
      "コード実装": 87.5,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 87.5
     }
    }
   }
  },
  "shitate/orchestrator": {
   "id": "shitate/orchestrator",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 97.4,
     "cost_usd": 0.3123,
     "median_s": 1.21,
     "p90_s": 4.24,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 97.2,
      "計算(表参照)": 95.9,
      "記憶(法定値)": 98.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 100.0,
     "cost_usd": 0.7395,
     "median_s": 2.01,
     "p90_s": 6.61,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 100.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 100.0,
     "cost_usd": 0.8359,
     "median_s": 1.34,
     "p90_s": 4.76,
     "errors": 0,
     "by_kind": {
      "コード実装": 100.0,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 100.0
     }
    }
   }
  },
  "thinkingmachines/inkling": {
   "id": "thinkingmachines/inkling",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 97.8,
     "cost_usd": 0.0999,
     "median_s": 2.75,
     "p90_s": 11.33,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 100.0,
      "計算": 97.5,
      "計算(表参照)": 95.9,
      "記憶(法定値)": 100.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 99.6,
     "cost_usd": 0.2128,
     "median_s": 1.27,
     "p90_s": 3.52,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 100.0,
      "指示追従": 98.5,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 95.7,
     "cost_usd": 0.3243,
     "median_s": 1.6,
     "p90_s": 3.28,
     "errors": 0,
     "by_kind": {
      "コード実装": 81.2,
      "多段推論(難)": 100.0,
      "長文照合": 100.0,
      "長文集計": 93.8
     }
    }
   }
  },
  "teai/auto": {
   "id": "teai/auto",
   "results": {
    "jp-business": {
     "n": 546,
     "items": 182,
     "accuracy": 86.3,
     "cost_usd": 0.0155,
     "median_s": 2.5,
     "p90_s": 5.6,
     "errors": 0,
     "by_kind": {
      "知識(仕様)": 94.4,
      "計算": 90.6,
      "計算(表参照)": 78.1,
      "記憶(法定値)": 82.0
     }
    },
    "general": {
     "n": 238,
     "items": 119,
     "accuracy": 97.8,
     "cost_usd": 0.0206,
     "median_s": 1.4,
     "p90_s": 3.7,
     "errors": 0,
     "by_kind": {
      "コード生成": 100.0,
      "多段推論": 94.1,
      "指示追従": 97.0,
      "長文抽出": 100.0
     }
    },
    "hard": {
     "n": 92,
     "items": 46,
     "accuracy": 96.2,
     "cost_usd": 0.0524,
     "median_s": 2.9,
     "p90_s": 12.2,
     "errors": 0,
     "by_kind": {
      "コード実装": 100.0,
      "多段推論(難)": 97.2,
      "長文照合": 100.0,
      "長文集計": 87.5
     }
    }
   }
  }
 },
 "prices_usd_per_1m": {
  "claude-fable-5": {
   "input": 10.0,
   "output": 50.0
  },
  "claude-sonnet-5": {
   "input": 3.0,
   "output": 15.0
  },
  "claude-haiku-4-5-20251001": {
   "input": 1.0,
   "output": 5.0
  },
  "openai/gpt-5.6-sol": {
   "input": 5.0,
   "output": 30.0
  },
  "x-ai/grok-4.5": {
   "input": 2.0,
   "output": 6.0
  },
  "moonshotai/kimi-k3": {
   "input": 3.0,
   "output": 15.0
  },
  "deepseek/deepseek-v4-pro": {
   "input": 0.43,
   "output": 0.87
  },
  "deepseek/deepseek-v4-flash-0731": {
   "input": 0.09,
   "output": 0.18
  },
  "z-ai/glm-5.2": {
   "input": 0.91,
   "output": 2.85
  },
  "qwen/qwen3.5-397b-a17b": {
   "input": 0.39,
   "output": 2.34
  },
  "google/gemini-3.1-pro-preview": {
   "input": 2.0,
   "output": 12.0
  },
  "sakana/fugu-ultra": {
   "input": 5.0,
   "output": 30.0
  },
  "shitate/orchestrator": {
   "input": 3.0,
   "output": 9.0
  },
  "thinkingmachines/inkling": {
   "input": 1.0,
   "output": 4.05
  }
 },
 "orchestration": {
  "eval": "jp-business",
  "configs_evaluated": 1820,
  "how": "各モデルの回答テキストを1回集めておけば、束ね方の評価に追加のAPI呼び出しは要らない。選定は片方の実行回だけで行い、もう片方(選定に使っていない回)で検証した。",
  "chosen": {
   "shape": "安い2本を並列 → 一致し、根拠が設問文にあり、検算で否定されなければ採用 → だめなら裁定役 → それでも合わなければ ⚠️要確認",
   "a": "deepseek/deepseek-v4-flash-0731",
   "b": "z-ai/glm-5.2",
   "arbiter": "thinkingmachines/inkling",
   "measured_live": [
    {
     "run": 1,
     "accuracy": 99.5,
     "flagged": 1.1,
     "silent_wrong": 0.0,
     "cost_usd": 0.173
    },
    {
     "run": 2,
     "accuracy": 100.0,
     "flagged": 1.1,
     "silent_wrong": 0.0,
     "cost_usd": 0.176
    },
    {
     "run": 3,
     "accuracy": 100.0,
     "flagged": 1.65,
     "silent_wrong": 0.0,
     "cost_usd": 0.168
    }
   ],
   "previous_arbiter": "google/gemini-3.1-pro-preview",
   "previous_cost_usd": 0.34
  },
  "rejected_by_holdout": {
   "shape": "安い1本を投げ、必要なときだけ上げる(裁定役の答えを突き合わせない)",
   "selection_run": {
    "accuracy": 100.0,
    "flagged": 0.0,
    "silent_wrong": 0.0
   },
   "holdout_run": {
    "accuracy": 98.9,
    "flagged": 0.0,
    "silent_wrong": 1.1
   },
   "why": "裁定役の答えを他と突き合わせずに受け入れる構造なので、裁定役が外れるとそのまま出る。選定回だけ見ていたら採用していた。"
  },
  "top_by_cost_with_zero_silent": [
   {
    "a": "3本",
    "b": "claude-haiku-4-5-20251001+deepseek/deepseek-v4-flash-0731",
    "s": "deepseek/deepseek-v4-pro",
    "n": 182,
    "acc": 88.46153846153847,
    "flag": 33.51648351648352,
    "silent": 0.0,
    "cost": 0.08880066999999997,
    "p90": 13.87
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "deepseek/deepseek-v4-pro",
    "s": "z-ai/glm-5.2",
    "n": 182,
    "acc": 98.35164835164835,
    "flag": 8.241758241758241,
    "silent": 0.0,
    "cost": 0.09069345000000001,
    "p90": 23.61
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "deepseek/deepseek-v4-pro",
    "s": "thinkingmachines/inkling",
    "n": 182,
    "acc": 99.45054945054945,
    "flag": 7.6923076923076925,
    "silent": 0.0,
    "cost": 0.09783846999999996,
    "p90": 18.53
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "deepseek/deepseek-v4-pro",
    "s": "qwen/qwen3.5-397b-a17b",
    "n": 182,
    "acc": 95.6043956043956,
    "flag": 10.989010989010989,
    "silent": 0.0,
    "cost": 0.10192687999999996,
    "p90": 26.92
   },
   {
    "a": "claude-haiku-4-5-20251001",
    "b": "deepseek/deepseek-v4-flash-0731",
    "s": "z-ai/glm-5.2",
    "n": 182,
    "acc": 96.15384615384616,
    "flag": 16.483516483516482,
    "silent": 0.0,
    "cost": 0.12883567000000004,
    "p90": 13.4
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "thinkingmachines/inkling",
    "s": "deepseek/deepseek-v4-pro",
    "n": 182,
    "acc": 93.4065934065934,
    "flag": 8.791208791208792,
    "silent": 0.0,
    "cost": 0.13336168000000004,
    "p90": 12.77
   },
   {
    "a": "claude-haiku-4-5-20251001",
    "b": "deepseek/deepseek-v4-flash-0731",
    "s": "thinkingmachines/inkling",
    "n": 182,
    "acc": 100.0,
    "flag": 13.736263736263735,
    "silent": 0.0,
    "cost": 0.13692343999999987,
    "p90": 5.69
   },
   {
    "a": "段階",
    "b": "thinkingmachines/inkling",
    "s": "z-ai/glm-5.2",
    "n": 182,
    "acc": 100.0,
    "flag": 0.0,
    "silent": 0.0,
    "cost": 0.1444904300000001,
    "p90": 7.33
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "z-ai/glm-5.2",
    "s": "deepseek/deepseek-v4-pro",
    "n": 182,
    "acc": 93.4065934065934,
    "flag": 9.340659340659341,
    "silent": 0.0,
    "cost": 0.14546078999999998,
    "p90": 18.99
   },
   {
    "a": "deepseek/deepseek-v4-pro",
    "b": "thinkingmachines/inkling",
    "s": "deepseek/deepseek-v4-flash-0731",
    "n": 182,
    "acc": 89.01098901098901,
    "flag": 12.087912087912088,
    "silent": 0.0,
    "cost": 0.15659123000000005,
    "p90": 15.52
   },
   {
    "a": "段階",
    "b": "thinkingmachines/inkling",
    "s": "openai/gpt-5.6-sol",
    "n": 182,
    "acc": 100.0,
    "flag": 0.0,
    "silent": 0.0,
    "cost": 0.15665490000000004,
    "p90": 4.27
   },
   {
    "a": "3本",
    "b": "deepseek/deepseek-v4-flash-0731+deepseek/deepseek-v4-pro",
    "s": "thinkingmachines/inkling",
    "n": 182,
    "acc": 92.85714285714286,
    "flag": 25.824175824175825,
    "silent": 0.0,
    "cost": 0.15797057,
    "p90": 13.9
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "thinkingmachines/inkling",
    "s": "z-ai/glm-5.2",
    "n": 182,
    "acc": 98.9010989010989,
    "flag": 1.098901098901099,
    "silent": 0.0,
    "cost": 0.16361079000000006,
    "p90": 9.46
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "deepseek/deepseek-v4-pro",
    "s": "openai/gpt-5.6-sol",
    "n": 182,
    "acc": 100.0,
    "flag": 7.142857142857143,
    "silent": 0.0,
    "cost": 0.16390066999999994,
    "p90": 17.22
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "thinkingmachines/inkling",
    "s": "thinkingmachines/inkling",
    "n": 182,
    "acc": 99.45054945054945,
    "flag": 0.5494505494505495,
    "silent": 0.0,
    "cost": 0.16470909000000006,
    "p90": 5.04
   },
   {
    "a": "deepseek/deepseek-v4-pro",
    "b": "z-ai/glm-5.2",
    "s": "deepseek/deepseek-v4-flash-0731",
    "n": 182,
    "acc": 88.46153846153847,
    "flag": 12.637362637362637,
    "silent": 0.0,
    "cost": 0.1676273900000001,
    "p90": 18.99
   },
   {
    "a": "claude-haiku-4-5-20251001",
    "b": "deepseek/deepseek-v4-pro",
    "s": "z-ai/glm-5.2",
    "n": 182,
    "acc": 96.15384615384616,
    "flag": 13.736263736263735,
    "silent": 0.0,
    "cost": 0.16779296000000002,
    "p90": 25.3
   },
   {
    "a": "deepseek/deepseek-v4-flash-0731",
    "b": "deepseek/deepseek-v4-pro",
    "s": "shitate/orchestrator",
    "n": 182,
    "acc": 97.8021978021978,
    "flag": 9.340659340659341,
    "silent": 0.0,
    "cost": 0.16889566999999986,
    "p90": 16.64
   },
   {
    "a": "3本",
    "b": "deepseek/deepseek-v4-flash-0731+deepseek/deepseek-v4-pro",
    "s": "z-ai/glm-5.2",
    "n": 182,
    "acc": 91.75824175824175,
    "flag": 26.373626373626372,
    "silent": 0.0,
    "cost": 0.16896191,
    "p90": 17.82
   },
   {
    "a": "3本",
    "b": "claude-haiku-4-5-20251001+deepseek/deepseek-v4-flash-0731",
    "s": "thinkingmachines/inkling",
    "n": 182,
    "acc": 92.3076923076923,
    "flag": 28.571428571428573,
    "silent": 0.0,
    "cost": 0.17408534000000006,
    "p90": 4.43
   }
  ]
 },
 "caveats": [
  "1構成あたりの標本は 182問 × 2回 = 364件。「364件で0件」が抑えるのは真の発生率およそ0.8%まで(95%)。0%の証明ではない。",
  "評価セットは自作で、いずれも特定の傾向を持つ。一般の性能ではない。",
  "自社の評価セットに正解側の誤りが3件あった(源泉徴収の二段階税率・割増賃金の端数処理・容積率の切り捨て)。いずれも**実務を正しく知っているモデルほど不正解になる**形だった。修正済み。",
  "難しい版のコード採点は、モデルが書いたコードを実際に実行している(別プロセス・作業ディレクトリ隔離・時間制限つき)。",
  "価格は上流のカタログ単価で、変動する。速度は測定時点のもの。",
  "system プロンプトの A/B は 182問 × 2回 = 364件/条件。±1〜2pt は誤差の範囲で、その幅の差を根拠にしていない。"
 ],
 "prompt_experiments": {
  "eval": "jp-business",
  "question": "system プロンプトを変えれば、安いモデルの正答率を汎用的に上げられるか。",
  "answer": "上げられない。同じ指示がモデルによって逆に働く。効くのは『そのモデルが実際に持っている手順の穴』に当てた場合だけ。",
  "variants": {
   "v0_baseline": "現行(答えだけ簡潔に述べる)",
   "v1_enumerate": "日付や個数は1つずつ書き出してから数える",
   "v2_rounding": "端数処理は指示された桁と方向を書き写してから適用する",
   "v3_quote_table": "設問文に表があれば使う行を引用してから答える",
   "v4_stages": "多段の計算は各段の結果を1行ずつ書く",
   "v5_all": "v1〜v4 を全部",
   "v6_all_abstain": "v5 + 記憶に頼る値は確信が無ければ『不明』と書く",
   "v7_care_only": "手順は指示せず『急がず見直せ』とだけ言う"
  },
  "results": {
   "deepseek/deepseek-v4-flash-0731": {
    "v0_baseline": {
     "accuracy": 93.4,
     "delta_vs_baseline": 0.0,
     "date_tasks": 78.6,
     "other_tasks": 96.1,
     "n": 364
    },
    "v1_enumerate": {
     "accuracy": 96.4,
     "delta_vs_baseline": 3.0,
     "date_tasks": 98.2,
     "other_tasks": 96.1,
     "n": 364
    },
    "v2_rounding": {
     "accuracy": 96.7,
     "delta_vs_baseline": 3.3,
     "date_tasks": 96.4,
     "other_tasks": 96.8,
     "n": 364
    },
    "v3_quote_table": {
     "accuracy": 86.8,
     "delta_vs_baseline": -6.6,
     "date_tasks": 58.9,
     "other_tasks": 91.9,
     "n": 364
    },
    "v4_stages": {
     "accuracy": 94.5,
     "delta_vs_baseline": 1.1,
     "date_tasks": 89.3,
     "other_tasks": 95.5,
     "n": 364
    },
    "v5_all": {
     "accuracy": 95.6,
     "delta_vs_baseline": 2.2,
     "date_tasks": 98.2,
     "other_tasks": 95.1,
     "n": 364
    },
    "v6_all_abstain": {
     "accuracy": 96.2,
     "delta_vs_baseline": 2.7,
     "date_tasks": 98.2,
     "other_tasks": 95.8,
     "n": 364
    },
    "v7_care_only": {
     "accuracy": 86.3,
     "delta_vs_baseline": -7.1,
     "date_tasks": 44.6,
     "other_tasks": 93.8,
     "n": 364
    }
   },
   "thinkingmachines/inkling": {
    "v0_baseline": {
     "accuracy": 98.1,
     "delta_vs_baseline": 0.0,
     "date_tasks": 94.6,
     "other_tasks": 98.7,
     "n": 364
    },
    "v1_enumerate": {
     "accuracy": 99.5,
     "delta_vs_baseline": 1.4,
     "date_tasks": 96.4,
     "other_tasks": 100.0,
     "n": 364
    },
    "v2_rounding": {
     "accuracy": 98.9,
     "delta_vs_baseline": 0.8,
     "date_tasks": 96.4,
     "other_tasks": 99.4,
     "n": 364
    },
    "v3_quote_table": {
     "accuracy": 98.9,
     "delta_vs_baseline": 0.8,
     "date_tasks": 100.0,
     "other_tasks": 98.7,
     "n": 364
    },
    "v4_stages": {
     "accuracy": 99.2,
     "delta_vs_baseline": 1.1,
     "date_tasks": 100.0,
     "other_tasks": 99.0,
     "n": 364
    },
    "v5_all": {
     "accuracy": 99.2,
     "delta_vs_baseline": 1.1,
     "date_tasks": 96.4,
     "other_tasks": 99.7,
     "n": 364
    },
    "v6_all_abstain": {
     "accuracy": 98.6,
     "delta_vs_baseline": 0.5,
     "date_tasks": 94.6,
     "other_tasks": 99.4,
     "n": 364
    },
    "v7_care_only": {
     "accuracy": 99.5,
     "delta_vs_baseline": 1.4,
     "date_tasks": 100.0,
     "other_tasks": 99.4,
     "n": 364
    }
   },
   "z-ai/glm-5.2": {
    "v0_baseline": {
     "accuracy": 96.7,
     "delta_vs_baseline": 0.0,
     "date_tasks": 94.6,
     "other_tasks": 97.1,
     "n": 364
    },
    "v1_enumerate": {
     "accuracy": 94.8,
     "delta_vs_baseline": -1.9,
     "date_tasks": 92.9,
     "other_tasks": 95.1,
     "n": 364
    },
    "v2_rounding": {
     "accuracy": 94.5,
     "delta_vs_baseline": -2.2,
     "date_tasks": 94.6,
     "other_tasks": 94.5,
     "n": 364
    },
    "v3_quote_table": {
     "accuracy": 95.9,
     "delta_vs_baseline": -0.8,
     "date_tasks": 98.2,
     "other_tasks": 95.5,
     "n": 364
    },
    "v4_stages": {
     "accuracy": 93.7,
     "delta_vs_baseline": -3.0,
     "date_tasks": 96.4,
     "other_tasks": 93.2,
     "n": 364
    },
    "v5_all": {
     "accuracy": 95.3,
     "delta_vs_baseline": -1.4,
     "date_tasks": 98.2,
     "other_tasks": 94.8,
     "n": 364
    },
    "v6_all_abstain": {
     "accuracy": 96.4,
     "delta_vs_baseline": -0.3,
     "date_tasks": 100.0,
     "other_tasks": 95.8,
     "n": 364
    },
    "v7_care_only": {
     "accuracy": 98.6,
     "delta_vs_baseline": 1.9,
     "date_tasks": 98.2,
     "other_tasks": 98.7,
     "n": 364
    }
   }
  },
  "headline_findings": [
   "deepseek-v4-flash の『祝日表つき営業日』は 58% → 95%(+37pt)。日付以外の設問への影響は ±0。列挙させれば数え間違いは直る。",
   "同じ『急がず見直せ』(v7)が、flash では日付設問を 58% → 0% に壊し、glm-5.2 では全体 +1.9 になった。指示の善し悪しはモデル依存。",
   "inkling と glm-5.2 は元から96〜100%で伸びしろが無く、手順を足すとむしろ下がった(glm-5.2 は v1〜v6 すべて負)。"
  ],
  "did_not_ship": {
   "what": "列挙の指示を :sure の安いモデルに入れる",
   "why": "**部品単体は良くなったのに、束ねた全体は悪くなった**。列挙させると出力が長く・形が揺れるため、もう1本との一致が減り、裁定役への持ち上げが増える。",
   "measured": [
    {
     "config": "現行(全部baseline)",
     "accuracy": "98.4〜100%",
     "flagged": "0.55〜3.30%",
     "cost_usd": "0.171〜0.183",
     "p90_s": "6.4〜11.7"
    },
    {
     "config": "flash だけ列挙",
     "accuracy": "98.4〜99.5%",
     "flagged": "3.30〜5.49%",
     "cost_usd": "0.206〜0.227",
     "p90_s": "14.4〜14.7"
    },
    {
     "config": "3本とも列挙(B も inkling に)",
     "accuracy": "98.9〜99.5%",
     "flagged": "5.49〜8.79%",
     "cost_usd": "0.189〜0.204",
     "p90_s": "5.0〜13.2"
    }
   ],
   "conclusion": "本番は現行のまま据え置いた。"
  }
 },
 "auto_router_note": {
  "description": "teai/auto は V4 Flash で回答し、コード検算・ルールベース・構造化チェックで低品質と判定された場合だけ V4 Pro にエスカレーションする。このデータは Flash と Pro の実測値からエスカレーション比率を推定して合成したもの。",
  "flash_model": "deepseek/deepseek-v4-flash-0731",
  "pro_model": "deepseek/deepseek-v4-pro",
  "escalation_rules": [
   "コード検算(verifier)でRejected → Pro",
   "回避表現・空回答・短すぎ → Pro",
   "JSONパース失敗・コードブロック未閉じ → Pro",
   "知識キーワード(法律・年収・契約) → Pro"
  ]
 }
}