{ "schema_version": 2, "label": "Corrected IFEval30: post-hoc common-27 reference comparison; protocol violations; no ranking", "manifest": { "seed": 20260924, "population_count": 541, "sampling": "sort key lexicographically as str, random.Random(seed).sample(population,30); frozen before inference", "records": [ { "problem_id": "1094", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "54d975ba4e32a83b4e333c2f972a08869dddd4d311ce45b1292a93c9eace57b9" }, { "problem_id": "1793", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "cbdf7a32889790e1df901b37b03c5111a828e8c4c70a4f77087185318bde53ff" }, { "problem_id": "13", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "c1da543fd9a3216d0390d1f9d50d967bd6c45f5807fe4f3df5cefe44d4c01f12" }, { "problem_id": "3305", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "00ad85de4b45e70fa426ce5c89d64fb84e2a504a9633260ff68c74e10a54e963" }, { "problem_id": "2471", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "8538da271863649effa642cfd8755c8ba387333e1e317e6a369659616b071df6" }, { "problem_id": "2941", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "60d6eeac938b2643d8b1146c3f995a676c0ee93a74a844100330e020e3dea166" }, { "problem_id": "3369", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "98d66c76d304832145a8951ed88d79248e238159652163783892bae0b74dcaf2" }, { "problem_id": "279", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "645189f964a1c0a0d2d06858a3b7a6614a2b917f8475e3791276d2140d6c388b" }, { "problem_id": "2832", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "f5e4c4f0a50f01da9a70287eb7187cef5251f6af0afb12a01c5be2ef4bcab75d" }, { "problem_id": "3367", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "a7159a1df324c1586584e948c3790d1e1375702dd20e0507221ad3e1f6040892" }, { "problem_id": "2341", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "c9e11ff8f9799ee5771e2854bfedf476ddb944ec6870a0eb4e84fee990815eae" }, { "problem_id": "1300", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "a52f6d8a2bba645dc47a2df162d60e455971344b062c68dcb4462607081c7db3" }, { "problem_id": "2416", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "54f87d32c7823aea83e923b4551d26faac76fd4d01efac4185fdd90667c99f71" }, { "problem_id": "2169", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "8fb53768d70fc4fee04c535577a5ef4af2bc4b25d8fc3af063ebdd2e949a1d14" }, { "problem_id": "2422", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "fe3b88033c092a0b91a9ec2b970f640467361bf94b9fc97ea4e89d4dcf364fc7" }, { "problem_id": "1203", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "ddbd4886b5dff0a4799c10c074e202a591a699c3189f6edd2cbe51c2112d0f02" }, { "problem_id": "1480", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "97b6ebfe7db3279237ca243446bed8152a3994a576317876250f7b5d5ac9f9ca" }, { "problem_id": "295", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "68776793fe37ed3a35beda4334b0a6e72bff46bb6a0ccf1cecb946f106a207f0" }, { "problem_id": "3494", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "854d587c5b54edf79e6911904cb393b64e7815ce4273d110fbb613f16467fdf8" }, { "problem_id": "2751", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "c734ee77d5d525d9e7e3de786413be221ac714f024de7dcf4568ba9f731da068" }, { "problem_id": "1392", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "7e923207f8fc37edc5352275799e826daef8ac595d011814ecd14e70ba1c2057" }, { "problem_id": "2386", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "651b20ad947e6e43588eaa40bee6eebb257e882b7dbc4d4561d9b59d02f399bc" }, { "problem_id": "3453", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "76426fdcfa4ad551d8519a06795ce081616abdf13155d4f0fc99c348bc2862e1" }, { "problem_id": "2243", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "1b79e0d9d167d1c7956cd5dc9d910c7f6de75f287fc3098f3905f946de0cda1a" }, { "problem_id": "3680", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "2eabb54e4781a4ce8e48956a6b6c469bad785247d07b88600226575596d0f5af" }, { "problem_id": "3623", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "97472493da813fcf5ac87379dfdbb6f07e39c405957d53e4f8037d513bb142b9" }, { "problem_id": "3434", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "714f9dbe06450501ff4fd4b9ced6c6625797e32863d347f4b740989c0df3561c" }, { "problem_id": "3414", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "1cefd2ca1f63a6f81bac2d1932d9e2b24cd4547e869525b5e8c8f9e060a5ab40" }, { "problem_id": "3326", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "36e4833958aac5946195ada9624bcf3348c5add6b4bac00dbea3dae2198a2d3b" }, { "problem_id": "3376", "commit": "d36068b845da4c2b24927fee2cea1e6ef98dadda", "source_sha256": "67ffeee0fcb87c317c5b08a2de85557b4a7e96ada6178aa645b4954fe4b53d49", "row_sha256": "617d0eafa2cbc65f16b03e309cd6b358035f3a3308da585aa908660314de6399" } ], "source_files": { "cost_estimate.json": { "sha256": "3a7feb98cb7a2b37073c6fcfd01da234c3b883a6b899c1df2abe1b52e2e92865", "bytes": 2381 }, "evaluation_plan.json": { "sha256": "50e003ab57f37763c76ccd6cfc84735f7a2fe1b51614134d56642087d3f9bfe4", "bytes": 17321 }, "ifeval30_alternative.json": { "sha256": "5020fe93e180f46088b9525877ac05ee2d38dfc36f6ee72df817307d3e42894a", "bytes": 51123 }, "responses_astra.json": { "sha256": "0f497699026a61e95c2a373894567ee6c7f4617905253f36b9b55e7a312c5abd", "bytes": 46777 }, "responses_claude-opus-5.5.json": { "sha256": "6ffb739527a1a9be4efe5a7961f874837699a41408aefa484169da33f8f9fb8f", "bytes": 67409 }, "responses_gpt-6-astra.json": { "sha256": "c4afd033384cbcb735c631cc5d95780444cf40dd3e77653b3621d161938fd660", "bytes": 46776 }, "responses_opus55.json": { "sha256": "a93593eb6a490d5c20388454c69cc37991c0fdc0a059dd03f71c1199bdd07975", "bytes": 67410 }, "responses_pareto.json": { "sha256": "ea383d31425a06eb1b1b696e411f182671d5fa37dde7aac0ea939f4ed32db2ce", "bytes": 37017 }, "scores_astra.json": { "sha256": "7918885c26ae21aada0fbf3fc5e784c7c28df633de489ec8192c5faf23cd8081", "bytes": 8445 }, "scores_claude-opus-5.5.json": { "sha256": "b2b5eee7024b6c6cfff62a1829ef03154120dceca6cd29fd820f973ee7c8a1a2", "bytes": 7688 }, "scores_gpt-6-astra.json": { "sha256": "57ea4256e85a8a238f88fedb7531fae045c6bd45854020782f830656e103532d", "bytes": 7698 }, "scores_opus55.json": { "sha256": "33795c3f1e294cfafacd0ad5fd0b348e70d562dc191b9d0aedf2ee4dd5fbca2f", "bytes": 8442 }, "scores_pareto.json": { "sha256": "35756008c782f7fa8028655abf69a05882c5cc99283caac4723cb708869ffa40", "bytes": 8822 }, "summary_ifeval30_20260924.json": { "sha256": "f9cc008a7b1b575a2d654aa3af4ba0d7ae4a60c0b3f24caab29efb385f334115", "bytes": 4190 }, "ledger_ifeval30_20260924.jsonl": { "sha256": "82c5e24df3b141763657c812e7dbf50f652ce48941f283c7b0991affba054fe6", "bytes": 208136 }, "run_ifeval30.py": { "sha256": "6dcce8ecf867ed87740f4a4e8623f54691dde8fccf7c17ad08c60bc765b7877e", "bytes": 13255 } }, "evaluation_plan_handling": "Has a trailing comma; retained as inert bytes for provenance, not executed or interpreted as results." }, "attribution": { "creator": "Google Research", "work": "IFEval / instruction_following_eval", "data_license": "CC-BY-4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", "source_url": "https://raw.githubusercontent.com/google-research/google-research/d36068b845da4c2b24927fee2cea1e6ef98dadda/instruction_following_eval/data/input_data.jsonl", "code_license": "Apache-2.0", "changes": "Independently selected 30-question subset; prompts not redistributed; responses selectively withheld. Published response strings are unchanged model outputs, not licensed source prompts." }, "protocol": { "questions": 30, "completions_per_model_question": 1, "seed": 20260924, "temperature_requested": 0, "temperature_gateway_omits_for": [ "anthropic/claude-opus-5.5" ], "max_tokens": 4096, "ttft_measured": false, "grader": "Official strict/loose judgments retained unchanged; missing responses are unknown in current analysis, not model failures.", "client_retries": 0, "gateway_max_attempts_same_model": 2 }, "sanitization": { "policy": "Fail-closed reviewed text allowlist plus sensitive-pattern scan; all other raw text withheld. No HTTP envelopes, credentials, headers, prompts, request IDs or local paths are exported.", "raw_response_scope": "Only final model response strings for problem IDs 279 and 3376 if review hashes match; illustrative selection, not representative performance examples.", "automated_scan_limit": "Pattern scanning alone cannot establish absence of personal or secret information. Only manually content-reviewed strings can pass the exact-text allowlist." }, "limitations": { "scope": { "ja": "IFEval公開問題から独立に抽出した30問のサブセットです。公式フルIFEvalでも、総合ランキングでもありません。コード能力は未測定です。", "en": "An independently sampled 30-question subset of public IFEval, not full official IFEval or an overall leaderboard. Coding ability was not measured." }, "contamination": { "ja": "IFEvalは2023年公開のデータです。評価モデルの学習への混入・汚染は未知であり、汚染なしとは主張しません。seed固定は学習汚染を防ぎません。", "en": "IFEval has been public since 2023. Training exposure and contamination are unknown; this is not a contamination-free claim. A fixed seed does not prevent training contamination." }, "publication": { "ja": "生応答は短い2問×3モデルだけを内容確認・完全一致ハッシュ固定して収録します。個人情報・秘密情報の疑い、未レビュー、ハッシュ不一致は原文を除外し、判定要約のみ残します。非公開の原文があるため、公開JSONだけで全問を再採点することはできません。", "en": "Raw text is limited to two short questions across three models, content-reviewed and pinned by exact hash. Suspected personal/secret data, unreviewed text or hash mismatches are withheld, leaving judgment summaries only. Because some raw text is withheld, the public JSON cannot reproduce grading for every question." }, "uncertainty": { "ja": "共通27問の事後比較です。信頼区間は選択偏りや測定手順違反を補正せず、統計的優位や順位は主張しません。", "en": "This is a post-hoc common-27 comparison. Confidence intervals do not correct selection bias or protocol violations; no statistical superiority or ranking is claimed." }, "timing": { "ja": "TTFTは未測定です。旧版の応答時間はHTTP完了までの通常応答集計で、同じ共通27問の再集計ではないため現行の主表から外しました。隠れた推論量も統一していません。", "en": "TTFT was not measured. Original latency aggregates covered complete HTTP normal responses, not the same common 27, so they are removed from the current main table. Hidden reasoning budgets were not normalized." }, "billing": { "ja": "費用は保存された返却usageと当時の単価からの推計で、確定請求・残高差分ではありません。未保存分や上流失敗をゼロ円とは扱いません。", "en": "Costs are estimates from saved returned usage and captured prices, not settled invoices or balance differences. Unsaved usage and upstream failures are not treated as zero cost." }, "source": { "ja": "現行集計は元scoresの問題別判定とledgerから独立再計算しました。生ledger・元scoresは不変です。旧summaryは撤回履歴だけに残し、現行数値の正本にはしません。", "en": "Current aggregates are independently recomputed from per-question source scores and the ledger. Raw ledger and source scores are unchanged. The old summary is retained only as withdrawn history, not as the source of current metrics." } }, "outcomes": { "anthropic/claude-opus-5.5": [ { "problem_id": "1094", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "fb773bde2f45e56d58ec1da3db44a00d3e7200b1fc8eb943c4377d9e660ddae2", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1793", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "25075d1f45e4373b9bb7bab16c2257cfd6a4784689286af589867772d8394d94", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "13", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "f49c99ff722af6d3b62988b207806684ffb3d1f2159b3d091bc4ab46e13b5d02", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3305", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "12eeeda4b4b2994b33c2cbca61899e1ef129a20abb6be0ebe7dfe1d9353e61af", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2471", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "b01fc8e9551a3808ab34f77dea90b7a1f109adbb5cce99014bd21682543dfcbd", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2941", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "d6035e72353375e5ab382ac33a71589b1cfd8363e0f5a0cdad2e19a70cdb2ac7", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3369", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "5736bf3a02c61b5e380c418684addee9299a016db892a106004eb480cbeaded1", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "279", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "0bfa49dff0a7fab90e2eefc5ac8d8f0c2dd46c59f14cb818636d491c2cf04bdf", "publication": "reviewed_verbatim_model_response", "response": "<>\n\nold stone glen at dusk—\na lone gull gliding through fog,\ngreen grass drinks the rain" }, { "problem_id": "2832", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ false ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "0c80a78ab2ebbb25c4a934a9d2e273c631aa1645d26017f32788543bfc1f2b68", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3367", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "9555a13e7c2431188b15ca4a078af18657c345c636f81f8f2fe148b22224b63e", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2341", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "42b8dcea23b20308460ea64c6ce1ee13430062b23141411e72ec5c8f5bb0a4cc", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1300", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false, true ], "loose_prompt": false, "loose_instructions": [ true, false, true ], "response_sha256": "1c7efb3cd8d7f30d4f843b8ea4b5e6965322c9cdae6ec81714b6b6b9dc316db5", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2416", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "0a612afa8191419a1f7c026bfdd04bd254c53dd2a7e6f7b4d0e7dcab897bbcf9", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2169", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "3f6d543c2a02aa6c50bdda58bb8e9fd7b73be0859b0181e32d8d7110b5df9cbd", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2422", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "016a00a157478d5805458340880e8cd046483992120a4cf95f52e6e8475ebf88", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1203", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "329c121e798ab9cfdc6d1333c12693247b27994daee188cbe6d8823c904e3a45", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1480", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "8dee9840459412f711037fedbd8a67d0c996e8e10a653a6ddbfb826b00dca765", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "295", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "6214c01bc7b6597d8acc44418e868acfe258e2da6f8c571d6b616ad772a82bcc", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3494", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "4cc45f226756fce20eb1d850c0fdc9e08768a6f9b21f9e0b008b65db0b8dae68", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2751", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "f08fefd25e5ab9660d702c10b9df4dfbe1fb03648e514324534c740f37e4d7e3", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1392", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "db2ae27135f1f81875a93c89d7472e39e2cbf5b384a9c3f22e35a0e3a0c9a804", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2386", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "5a7b606629c3e97af58f9c3365a14a9aacce10093036fdf36fc0b31917adb920", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3453", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "446e8fe3a1315f51abb470d5998171217d32185f462585d0bd69f9edb807baa3", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2243", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "b227bf1c43119badc3ba54e9de6734a1853fb542eae2bd99657bfd672e1b2420", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3680", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "12d0397e642c1481393b22d2ea0147bfed98768ffa77b02b3ccf60f13d4ac41b", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3623", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "97f7198ea0ab55c408f6ed2b76b72929a2ddf942d467c9c065d2752649690af0", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3434", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "1687c73dd5acf87c06c07bdb922055d5f42a59e5f4ec333261b2d8ddd3b8739b", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3414", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "13bb781b0ff555a776303ad95dfd2160f34d785fb34836658261047894963784", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3326", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "fcaf26bf54164820a15ee94b45f45180f236a7e547167df820d4d99a3f28479b", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3376", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "ea3f41dfc469e8affaf2ac60d5e50365f5618b7c2edbbe470db513f4c01bc7f5", "publication": "reviewed_verbatim_model_response", "response": "**The Embroidery Riddle**\n\nI have one eye but cannot see\nAnd drag a thread that follows me.\nThrough linen stretched on wooden rings\nI dive and rise and sew fine things.\nAll afternoon I dart and gleam\nTo stitch a rose along a seam.\nIf I slip off and hide away\nThe stitcher grows distressed all day.\nWhat am I?\n\n**Answer:** A needle" } ], "openai/gpt-6-astra": [ { "problem_id": "1094", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "0987ebd5cfae7ac84fe21a7895b7198514bf0db96ca4ca9594c041936e15d0e5", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1793", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "891d18a8f706859b4057186f1acea7bac055b1440022ce23ec61271b8ec1fbba", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "13", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "ded5a36866e56d12e057b5defe83e1238d2ee3b48ed2b6e71d402ec1c1851f91", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3305", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false, false ], "loose_prompt": false, "loose_instructions": [ true, false, false ], "response_sha256": "27d9dc128a1bd3d9fc6ffc242fd987bcde2ddc2052b04e957c6111345f4430c9", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2471", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "a1ea335cd243993cc9e9bc2f767f1b5a5b1fef878bca2acba27cc0d1c5f85f88", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2941", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "78b6dd408583aff0c68b1b12376ffd75a53b4803d87f6afe9ac5d53dff618e1e", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3369", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "08502e0059dacccd320b92856f610c9149d1e75719453e0249769e6dd0bca606", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "279", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "c41cdf264734090c8848315d1ca8a6a41f9238fdd3700f861e1e0d2743cae74b", "publication": "reviewed_verbatim_model_response", "response": "<>\n\nrain on the bog road \na lone frog stirs the still pond \nspring fog veils the hills" }, { "problem_id": "2832", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "fd712f439474a00f7c840e5a0b6f18e1a9d0e9cf5aee5daf401cf63524de0540", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3367", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "ce33db65392a5f1964cae3fa295c02482b164a9318095c4187706a048473d83a", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2341", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "854c8b80546842660a14f4c9c37a20ae6667bec928bd93446c3591f51544e6f1", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1300", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false, true ], "loose_prompt": false, "loose_instructions": [ true, false, true ], "response_sha256": "84a4c528e011b14fab3626adcbeeb93f453930aca25aecf1022da92d6f8e29ac", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2416", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "436d8d39489688050e3366fc84c059184d6931db1e1ae663147c537eae00ade9", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2169", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "c9e61d69d6d29702ed094e231230631b21c18c60044bf50bfb67aeaa2f781a56", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2422", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ false, true ], "loose_prompt": false, "loose_instructions": [ false, true ], "response_sha256": "630e266b9ff9e773d8457b28b93fd425c9cec1d0f4c8801febd32fe0c55e9cff", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1203", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "350f9a0fea18649d103af6779a2e407683cd54bd204b81fbadca1ad7777acb17", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1480", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "860c65ad61e2c7d641e3e2fe1909368ac7d8c81fe1c804e6169532fe30ab852e", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "295", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "a993faf46064c29f10c208c740d1f4667b34dbe717726d5e705c7a1e844ffd23", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3494", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "3d408f459d5fdd1993b260558184b6b8aa71bab59812c3c7587f7e4c9263ed26", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2751", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "1601bf8636df98a59894f7acbfc57b344c9d1668741d27757e89d6ea6151bc14", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1392", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "260a71b225d7ccaa512f7e863812601ade2adedc88b6275f6f59ec4dcebc132c", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2386", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "acf544962e6b504dcb3917427672483ebb09bcd532e2f9cb3038a7ebaea48d44", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3453", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "b46b3cdae2eb890d8e3f021a89a5773d5d8d8f37e9424790dec22f4bc6005273", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2243", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "695c65802e20dd0034ce54df482d6ce8e65752a2b0204b0a57d3b21b1910cbd6", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3680", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "7f7d378e45312e483000592943218d5a499648ee648ba4b7aacc307b7f039e7b", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3623", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "a092541f70a74c50129733b99b733ae567deb933a40cc004ed2641e474448a6d", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3434", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ false, true ], "loose_prompt": false, "loose_instructions": [ false, true ], "response_sha256": "e4df46ddfcb7d9ddb74d0057d4f9c43a46d03c041c31e01272e55f8d0932cdc9", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3414", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "614e86dd07767cbf7eb488efcd18cf931cd705e58ef2b457dd25f76aaef86341", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3326", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "39314a4ee71f2cd9fb516b4a58a0a932e46898c6f7bf90de52a714dcfdbd04c6", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3376", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "7fd676a391c09e2d26f4f6bb0fc0a4c2bc752b0e82bdaf5e149f7732bad80af2", "publication": "reviewed_verbatim_model_response", "response": "I have one eye but cannot see.\nOn a quiet afternoon you embroider with me.\nIf I go missing you may feel distressed.\nWhat sharp little tool leaves your fabric flower-dressed?\n\nAnswer: needle" } ], "unbiased/pareto": [ { "problem_id": "1094", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "3c0fe42b00cc58534aa8b3979388bff560a79e996de8a8eedf15381f85f75516", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1793", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "a1e6f8e1fc1fb8a1d522ec6ce13f48277afc7eb8117972d45df58d87e0530e91", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "13", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "f85cd4a1a191c54f6ca3987688ea553c3230b48446730f2ae652c556940c2b83", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3305", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false, false ], "loose_prompt": false, "loose_instructions": [ true, false, false ], "response_sha256": "5dd8a07a7f4ccc52726657797725b55ad551bb2f6f0cc223cb20c07b80785b8a", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2471", "is_normal_answer": false, "strict_prompt": false, "strict_instructions": [ false, false ], "loose_prompt": false, "loose_instructions": [ false, false ], "response_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "publication": "missing_not_a_model_answer" }, { "problem_id": "2941", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "2ea0a3d576ac2447ab84d923cd74cb7aa6b4ce2fc154e261d1fec2a1e941bdc4", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3369", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "7deda475cb65548a0ea0972cd5b3ffbb76c67051882c3c47156eba39d75984a7", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "279", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "99c018f7454a7993f5762fc46251a44e100d4d59af2466090bd459b181cc2c6f", "publication": "reviewed_verbatim_model_response", "response": "<>\n\ngreen glens greet the rain \nbog grasses cling to old stones \ngulls glide through the fog" }, { "problem_id": "2832", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "1fd78989d44a2844e6fb8b8ef99572a6e81695e85b2a3cb6ce879767d7882a2e", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3367", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false ], "loose_prompt": false, "loose_instructions": [ true, false ], "response_sha256": "4decc10b3bf673d7474054d585993990b25033fd3fd921c381cb39e6a23f9ad8", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2341", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "9ae8ee47444a9c2332ad0f1c37582ddcd0d5197b119144ee34fbbafba9625d3e", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1300", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false, true ], "loose_prompt": false, "loose_instructions": [ true, false, true ], "response_sha256": "675c203b702545ce995d32bdf5999f814b774045c4c1391c720c5045650f7f70", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2416", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "86e11d43076205f4073914a386295aba981fba579f8e9eb1b0660b6beeb21c31", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2169", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "cde75ff325c0efeee2e81e064d0fa05ad282705f1ce40ee3c74c3cd7d958ee9f", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2422", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "3f54d60c406c2dfc0701220463ab6a9cff5aaafaa2bd909df4f713d6ebb9bfe3", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1203", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "6ef45e21dd1ad4094007dbd0b983afe2a7c9d9d2cf108b907e2fc73135d7ea26", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1480", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "5bd197b592408c096eee0b9286f24ae3270f2b35a55347bc0c9134cf44b74490", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "295", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "407902bf8905e730041967e48830bb1c45a356622d37d5b4c6ec0a0d92df72cb", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3494", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "a9a3ed7e068ba0fa64ad69101c4d0d6b19c40d32a4fccf27df1b583d4bf64a15", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2751", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "931f9f362cc83263323e2f2a3d8c7a56957f76c2ae40b88fe935d35eaaa18c8a", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "1392", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true, true ], "loose_prompt": true, "loose_instructions": [ true, true, true ], "response_sha256": "ba85486cdcc91b697de23be6c6b4e8b68a52ade91d18be9a36c3f600462a7f21", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2386", "is_normal_answer": false, "strict_prompt": false, "strict_instructions": [ false, false ], "loose_prompt": false, "loose_instructions": [ false, false ], "response_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "publication": "missing_not_a_model_answer" }, { "problem_id": "3453", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "7aeafd6fef5e60e1c4a45dda482b4f766bc3f9215e4c44478db3fece99f77ad7", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "2243", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true ], "loose_prompt": true, "loose_instructions": [ true ], "response_sha256": "e3f29ff77ed00a2286bd45be05a9967a004b061709bca5530a490147ab23aa22", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3680", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "c4192fbb33408b6837a85d40ada9bba228ede936b2066a3a0b72642159659a1e", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3623", "is_normal_answer": true, "strict_prompt": false, "strict_instructions": [ true, false ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "c756c8508d78177fd5889455892f803338ddf33bff96ed4d81d7ed1e3a399704", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3434", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "ac77c8ff28990c52f824565de2a13d98cacadc437f3c15b3ade3063ed09f0f03", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3414", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "562619f56763069451516c8447cf83984dec42f39e9a1ca6ae564ad6c792edb6", "publication": "withheld_not_reviewed_or_hash_changed" }, { "problem_id": "3326", "is_normal_answer": false, "strict_prompt": false, "strict_instructions": [ false, false ], "loose_prompt": false, "loose_instructions": [ false, false ], "response_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "publication": "missing_not_a_model_answer" }, { "problem_id": "3376", "is_normal_answer": true, "strict_prompt": true, "strict_instructions": [ true, true ], "loose_prompt": true, "loose_instructions": [ true, true ], "response_sha256": "3363c520a3a27bd91ded59e310708d13d0c72c95bae5cb9cdb5ad573a664920d", "publication": "reviewed_verbatim_model_response", "response": "I spend the afternoon slipping through seams.\nI turn distressed cloth into embroidered dreams.\nI have one eye but cannot see.\nWhat pointed little tool could I be?\n\n**Answer: needle**" } ] }, "current": { "common_ids": [ "1094", "1793", "13", "3305", "2941", "3369", "279", "2832", "3367", "2341", "1300", "2416", "2169", "2422", "1203", "1480", "295", "3494", "2751", "1392", "3453", "2243", "3680", "3623", "3434", "3414", "3376" ], "excluded_ids": [ "2471", "2386", "3326" ], "comparison": { "anthropic/claude-opus-5.5": { "problem_ids": [ "1094", "1793", "13", "3305", "2941", "3369", "279", "2832", "3367", "2341", "1300", "2416", "2169", "2422", "1203", "1480", "295", "3494", "2751", "1392", "3453", "2243", "3680", "3623", "3434", "3414", "3376" ], "strict_prompt": { "passed": 23, "total": 27 }, "loose_prompt": { "passed": 26, "total": 27 }, "strict_instructions": { "passed": 46, "total": 50 }, "loose_instructions": { "passed": 49, "total": 50 } }, "openai/gpt-6-astra": { "problem_ids": [ "1094", "1793", "13", "3305", "2941", "3369", "279", "2832", "3367", "2341", "1300", "2416", "2169", "2422", "1203", "1480", "295", "3494", "2751", "1392", "3453", "2243", "3680", "3623", "3434", "3414", "3376" ], "strict_prompt": { "passed": 22, "total": 27 }, "loose_prompt": { "passed": 23, "total": 27 }, "strict_instructions": { "passed": 44, "total": 50 }, "loose_instructions": { "passed": 45, "total": 50 } }, "unbiased/pareto": { "problem_ids": [ "1094", "1793", "13", "3305", "2941", "3369", "279", "2832", "3367", "2341", "1300", "2416", "2169", "2422", "1203", "1480", "295", "3494", "2751", "1392", "3453", "2243", "3680", "3623", "3434", "3414", "3376" ], "strict_prompt": { "passed": 22, "total": 27 }, "loose_prompt": { "passed": 24, "total": 27 }, "strict_instructions": { "passed": 44, "total": 50 }, "loose_instructions": { "passed": 46, "total": 50 } } }, "full30_strict": { "anthropic/claude-opus-5.5": { "correct": 26, "incorrect": 4, "unknown": 0 }, "openai/gpt-6-astra": { "correct": 25, "incorrect": 5, "unknown": 0 }, "unbiased/pareto": { "correct": 22, "incorrect": 5, "unknown": 3 } }, "cost": { "per_model": { "anthropic/claude-opus-5.5": { "saved_usage_count": 30, "estimated_jpy": "108.956988" }, "openai/gpt-6-astra": { "saved_usage_count": 30, "estimated_jpy": "92.385090" }, "unbiased/pareto": { "saved_usage_count": 27, "estimated_jpy": "16.035495" } }, "saved_usage_count": 87, "estimated_jpy": "217.377573", "unknown_rows": [ { "model": "unbiased/pareto", "problem_id": "2471", "result_line": 29 }, { "model": "unbiased/pareto", "problem_id": "2386", "result_line": null }, { "model": "unbiased/pareto", "problem_id": "3326", "result_line": 177 } ], "reservation_jpy": 2133, "formula": "(prompt_tokens * credits_in_1k + completion_tokens * credits_out_1k) / 6000", "prices": { "anthropic/claude-opus-5.5": { "credits_in_1k": "3.528", "credits_out_1k": "17.64" }, "openai/gpt-6-astra": { "credits_in_1k": "8.82", "credits_out_1k": "44.1" }, "unbiased/pareto": { "credits_in_1k": "2.205", "credits_out_1k": "6.615" } } }, "execution": { "reservations": 90, "results": 89, "abandoned": 1, "http_errors": 0, "config_events": [ { "line": 30, "event": "halt", "reason": "gateway identity mismatch", "at": "2026-09-24T03:10:47.293759+00:00" }, { "line": 32, "event": "config_revision", "reason": "identity rule corrected to served-model match only per task spec; fallback_depth recorded but not judged; earlier results remain valid under both rules", "at": "2026-09-24T03:13:29.088400+00:00" }, { "line": 136, "event": "config_revision", "reason": "identity rule corrected to served-model match only per task spec; fallback_depth recorded but not judged; earlier results remain valid under both rules", "at": "2026-09-24T03:28:42.232610+00:00" }, { "line": 178, "event": "halt", "reason": "usage completion_tokens exceeds max_tokens", "at": "2026-09-24T03:33:29.556970+00:00" }, { "line": 180, "event": "config_revision", "reason": "identity rule corrected to served-model match only per task spec; fallback_depth recorded but not judged; earlier results remain valid under both rules", "at": "2026-09-24T03:33:54.952555+00:00" } ], "usage_halt_line": 178, "post_halt_requests": [ { "model": "anthropic/claude-opus-5.5", "problem_id": "3326", "reserve_line": 181, "result_line": 182 }, { "model": "unbiased/pareto", "problem_id": "3376", "reserve_line": 183, "result_line": 184 }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3376", "reserve_line": 185, "result_line": 186 }, { "model": "openai/gpt-6-astra", "problem_id": "3376", "reserve_line": 187, "result_line": 188 } ], "restart_audit": { "source": "User-provided independent audit and read-only runner review", "cause_resolution_confirmed": false, "explicit_restart_approval": false, "persistent_halt": false }, "client_retries": 0, "gateway_max_attempts_same_model": 2, "failed_upstream_usage_known": false, "finish_reason_rewritten_to_stop": true, "truncation_verified": false, "served_proves_upstream_identity": false, "cap_guaranteed": false }, "source_rows": [ { "model": "anthropic/claude-opus-5.5", "problem_id": "1094", "reserve_line": 2, "result_line": 3, "response_saved": true, "usage": { "prompt_tokens": 108, "completion_tokens": 778 }, "estimated_jpy": "2.350824" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "1793", "reserve_line": 12, "result_line": 13, "response_saved": true, "usage": { "prompt_tokens": 163, "completion_tokens": 831 }, "estimated_jpy": "2.538984" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "13", "reserve_line": 16, "result_line": 17, "response_saved": true, "usage": { "prompt_tokens": 113, "completion_tokens": 2702 }, "estimated_jpy": "8.010324" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3305", "reserve_line": 20, "result_line": 21, "response_saved": true, "usage": { "prompt_tokens": 158, "completion_tokens": 1793 }, "estimated_jpy": "5.364324" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2471", "reserve_line": 33, "result_line": 34, "response_saved": true, "usage": { "prompt_tokens": 138, "completion_tokens": 3577 }, "estimated_jpy": "10.597524" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2941", "reserve_line": 37, "result_line": 38, "response_saved": true, "usage": { "prompt_tokens": 112, "completion_tokens": 653 }, "estimated_jpy": "1.985676" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3369", "reserve_line": 41, "result_line": 42, "response_saved": true, "usage": { "prompt_tokens": 120, "completion_tokens": 586 }, "estimated_jpy": "1.7934" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "279", "reserve_line": 51, "result_line": 52, "response_saved": true, "usage": { "prompt_tokens": 121, "completion_tokens": 407 }, "estimated_jpy": "1.267728" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2832", "reserve_line": 55, "result_line": 56, "response_saved": true, "usage": { "prompt_tokens": 135, "completion_tokens": 566 }, "estimated_jpy": "1.74342" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3367", "reserve_line": 59, "result_line": 60, "response_saved": true, "usage": { "prompt_tokens": 153, "completion_tokens": 715 }, "estimated_jpy": "2.192064" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2341", "reserve_line": 69, "result_line": 70, "response_saved": true, "usage": { "prompt_tokens": 84, "completion_tokens": 870 }, "estimated_jpy": "2.607192" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "1300", "reserve_line": 73, "result_line": 74, "response_saved": true, "usage": { "prompt_tokens": 140, "completion_tokens": 4096 }, "estimated_jpy": "12.12456" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2416", "reserve_line": 77, "result_line": 78, "response_saved": true, "usage": { "prompt_tokens": 135, "completion_tokens": 1240 }, "estimated_jpy": "3.72498" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2169", "reserve_line": 87, "result_line": 88, "response_saved": true, "usage": { "prompt_tokens": 107, "completion_tokens": 1090 }, "estimated_jpy": "3.267516" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2422", "reserve_line": 91, "result_line": 92, "response_saved": true, "usage": { "prompt_tokens": 123, "completion_tokens": 966 }, "estimated_jpy": "2.912364" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "1203", "reserve_line": 95, "result_line": 96, "response_saved": true, "usage": { "prompt_tokens": 100, "completion_tokens": 2096 }, "estimated_jpy": "6.22104" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "1480", "reserve_line": 105, "result_line": 106, "response_saved": true, "usage": { "prompt_tokens": 121, "completion_tokens": 1851 }, "estimated_jpy": "5.513088" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "295", "reserve_line": 109, "result_line": 110, "response_saved": true, "usage": { "prompt_tokens": 119, "completion_tokens": 1012 }, "estimated_jpy": "3.045252" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3494", "reserve_line": 113, "result_line": 114, "response_saved": true, "usage": { "prompt_tokens": 133, "completion_tokens": 1033 }, "estimated_jpy": "3.115224" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2751", "reserve_line": 123, "result_line": 124, "response_saved": true, "usage": { "prompt_tokens": 95, "completion_tokens": 1375 }, "estimated_jpy": "4.09836" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "1392", "reserve_line": 127, "result_line": 128, "response_saved": true, "usage": { "prompt_tokens": 132, "completion_tokens": 1596 }, "estimated_jpy": "4.769856" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2386", "reserve_line": 131, "result_line": 132, "response_saved": true, "usage": { "prompt_tokens": 134, "completion_tokens": 505 }, "estimated_jpy": "1.563492" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3453", "reserve_line": 142, "result_line": 143, "response_saved": true, "usage": { "prompt_tokens": 105, "completion_tokens": 918 }, "estimated_jpy": "2.76066" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "2243", "reserve_line": 146, "result_line": 147, "response_saved": true, "usage": { "prompt_tokens": 99, "completion_tokens": 427 }, "estimated_jpy": "1.313592" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3680", "reserve_line": 150, "result_line": 151, "response_saved": true, "usage": { "prompt_tokens": 109, "completion_tokens": 1530 }, "estimated_jpy": "4.562292" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3623", "reserve_line": 160, "result_line": 161, "response_saved": true, "usage": { "prompt_tokens": 110, "completion_tokens": 424 }, "estimated_jpy": "1.31124" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3434", "reserve_line": 164, "result_line": 165, "response_saved": true, "usage": { "prompt_tokens": 135, "completion_tokens": 798 }, "estimated_jpy": "2.4255" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3414", "reserve_line": 168, "result_line": 169, "response_saved": true, "usage": { "prompt_tokens": 106, "completion_tokens": 869 }, "estimated_jpy": "2.617188" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3326", "reserve_line": 181, "result_line": 182, "response_saved": true, "usage": { "prompt_tokens": 99, "completion_tokens": 701 }, "estimated_jpy": "2.119152" }, { "model": "anthropic/claude-opus-5.5", "problem_id": "3376", "reserve_line": 185, "result_line": 186, "response_saved": true, "usage": { "prompt_tokens": 109, "completion_tokens": 332 }, "estimated_jpy": "1.040172" }, { "model": "openai/gpt-6-astra", "problem_id": "1094", "reserve_line": 4, "result_line": 5, "response_saved": true, "usage": { "prompt_tokens": 70, "completion_tokens": 118 }, "estimated_jpy": "0.9702" }, { "model": "openai/gpt-6-astra", "problem_id": "1793", "reserve_line": 8, "result_line": 9, "response_saved": true, "usage": { "prompt_tokens": 105, "completion_tokens": 402 }, "estimated_jpy": "3.10905" }, { "model": "openai/gpt-6-astra", "problem_id": "13", "reserve_line": 18, "result_line": 19, "response_saved": true, "usage": { "prompt_tokens": 72, "completion_tokens": 743 }, "estimated_jpy": "5.56689" }, { "model": "openai/gpt-6-astra", "problem_id": "3305", "reserve_line": 22, "result_line": 23, "response_saved": true, "usage": { "prompt_tokens": 111, "completion_tokens": 688 }, "estimated_jpy": "5.21997" }, { "model": "openai/gpt-6-astra", "problem_id": "2471", "reserve_line": 26, "result_line": 27, "response_saved": true, "usage": { "prompt_tokens": 95, "completion_tokens": 1168 }, "estimated_jpy": "8.72445" }, { "model": "openai/gpt-6-astra", "problem_id": "2941", "reserve_line": 39, "result_line": 40, "response_saved": true, "usage": { "prompt_tokens": 74, "completion_tokens": 256 }, "estimated_jpy": "1.99038" }, { "model": "openai/gpt-6-astra", "problem_id": "3369", "reserve_line": 43, "result_line": 44, "response_saved": true, "usage": { "prompt_tokens": 90, "completion_tokens": 263 }, "estimated_jpy": "2.06535" }, { "model": "openai/gpt-6-astra", "problem_id": "279", "reserve_line": 47, "result_line": 48, "response_saved": true, "usage": { "prompt_tokens": 83, "completion_tokens": 304 }, "estimated_jpy": "2.35641" }, { "model": "openai/gpt-6-astra", "problem_id": "2832", "reserve_line": 57, "result_line": 58, "response_saved": true, "usage": { "prompt_tokens": 91, "completion_tokens": 231 }, "estimated_jpy": "1.83162" }, { "model": "openai/gpt-6-astra", "problem_id": "3367", "reserve_line": 61, "result_line": 62, "response_saved": true, "usage": { "prompt_tokens": 94, "completion_tokens": 139 }, "estimated_jpy": "1.15983" }, { "model": "openai/gpt-6-astra", "problem_id": "2341", "reserve_line": 65, "result_line": 66, "response_saved": true, "usage": { "prompt_tokens": 56, "completion_tokens": 128 }, "estimated_jpy": "1.02312" }, { "model": "openai/gpt-6-astra", "problem_id": "1300", "reserve_line": 75, "result_line": 76, "response_saved": true, "usage": { "prompt_tokens": 99, "completion_tokens": 574 }, "estimated_jpy": "4.36443" }, { "model": "openai/gpt-6-astra", "problem_id": "2416", "reserve_line": 79, "result_line": 80, "response_saved": true, "usage": { "prompt_tokens": 94, "completion_tokens": 616 }, "estimated_jpy": "4.66578" }, { "model": "openai/gpt-6-astra", "problem_id": "2169", "reserve_line": 83, "result_line": 84, "response_saved": true, "usage": { "prompt_tokens": 72, "completion_tokens": 487 }, "estimated_jpy": "3.68529" }, { "model": "openai/gpt-6-astra", "problem_id": "2422", "reserve_line": 93, "result_line": 94, "response_saved": true, "usage": { "prompt_tokens": 83, "completion_tokens": 141 }, "estimated_jpy": "1.15836" }, { "model": "openai/gpt-6-astra", "problem_id": "1203", "reserve_line": 97, "result_line": 98, "response_saved": true, "usage": { "prompt_tokens": 70, "completion_tokens": 740 }, "estimated_jpy": "5.5419" }, { "model": "openai/gpt-6-astra", "problem_id": "1480", "reserve_line": 101, "result_line": 102, "response_saved": true, "usage": { "prompt_tokens": 81, "completion_tokens": 525 }, "estimated_jpy": "3.97782" }, { "model": "openai/gpt-6-astra", "problem_id": "295", "reserve_line": 111, "result_line": 112, "response_saved": true, "usage": { "prompt_tokens": 76, "completion_tokens": 388 }, "estimated_jpy": "2.96352" }, { "model": "openai/gpt-6-astra", "problem_id": "3494", "reserve_line": 115, "result_line": 116, "response_saved": true, "usage": { "prompt_tokens": 92, "completion_tokens": 252 }, "estimated_jpy": "1.98744" }, { "model": "openai/gpt-6-astra", "problem_id": "2751", "reserve_line": 119, "result_line": 120, "response_saved": true, "usage": { "prompt_tokens": 61, "completion_tokens": 663 }, "estimated_jpy": "4.96272" }, { "model": "openai/gpt-6-astra", "problem_id": "1392", "reserve_line": 129, "result_line": 130, "response_saved": true, "usage": { "prompt_tokens": 95, "completion_tokens": 518 }, "estimated_jpy": "3.94695" }, { "model": "openai/gpt-6-astra", "problem_id": "2386", "reserve_line": 133, "result_line": 134, "response_saved": true, "usage": { "prompt_tokens": 93, "completion_tokens": 158 }, "estimated_jpy": "1.29801" }, { "model": "openai/gpt-6-astra", "problem_id": "3453", "reserve_line": 138, "result_line": 139, "response_saved": true, "usage": { "prompt_tokens": 71, "completion_tokens": 522 }, "estimated_jpy": "3.94107" }, { "model": "openai/gpt-6-astra", "problem_id": "2243", "reserve_line": 148, "result_line": 149, "response_saved": true, "usage": { "prompt_tokens": 68, "completion_tokens": 191 }, "estimated_jpy": "1.50381" }, { "model": "openai/gpt-6-astra", "problem_id": "3680", "reserve_line": 152, "result_line": 153, "response_saved": true, "usage": { "prompt_tokens": 73, "completion_tokens": 602 }, "estimated_jpy": "4.53201" }, { "model": "openai/gpt-6-astra", "problem_id": "3623", "reserve_line": 156, "result_line": 157, "response_saved": true, "usage": { "prompt_tokens": 75, "completion_tokens": 166 }, "estimated_jpy": "1.33035" }, { "model": "openai/gpt-6-astra", "problem_id": "3434", "reserve_line": 166, "result_line": 167, "response_saved": true, "usage": { "prompt_tokens": 84, "completion_tokens": 294 }, "estimated_jpy": "2.28438" }, { "model": "openai/gpt-6-astra", "problem_id": "3414", "reserve_line": 170, "result_line": 171, "response_saved": true, "usage": { "prompt_tokens": 71, "completion_tokens": 262 }, "estimated_jpy": "2.03007" }, { "model": "openai/gpt-6-astra", "problem_id": "3326", "reserve_line": 174, "result_line": 175, "response_saved": true, "usage": { "prompt_tokens": 67, "completion_tokens": 327 }, "estimated_jpy": "2.50194" }, { "model": "openai/gpt-6-astra", "problem_id": "3376", "reserve_line": 187, "result_line": 188, "response_saved": true, "usage": { "prompt_tokens": 71, "completion_tokens": 216 }, "estimated_jpy": "1.69197" }, { "model": "unbiased/pareto", "problem_id": "1094", "reserve_line": 6, "result_line": 7, "response_saved": true, "usage": { "prompt_tokens": 85, "completion_tokens": 114 }, "estimated_jpy": "0.1569225" }, { "model": "unbiased/pareto", "problem_id": "1793", "reserve_line": 10, "result_line": 11, "response_saved": true, "usage": { "prompt_tokens": 107, "completion_tokens": 1386 }, "estimated_jpy": "1.5673875" }, { "model": "unbiased/pareto", "problem_id": "13", "reserve_line": 14, "result_line": 15, "response_saved": true, "usage": { "prompt_tokens": 74, "completion_tokens": 837 }, "estimated_jpy": "0.9499875" }, { "model": "unbiased/pareto", "problem_id": "3305", "reserve_line": 24, "result_line": 25, "response_saved": true, "usage": { "prompt_tokens": 109, "completion_tokens": 699 }, "estimated_jpy": "0.810705" }, { "model": "unbiased/pareto", "problem_id": "2471", "reserve_line": 28, "result_line": 29, "response_saved": false, "usage": null, "estimated_jpy": null }, { "model": "unbiased/pareto", "problem_id": "2941", "reserve_line": 35, "result_line": 36, "response_saved": true, "usage": { "prompt_tokens": 72, "completion_tokens": 287 }, "estimated_jpy": "0.3428775" }, { "model": "unbiased/pareto", "problem_id": "3369", "reserve_line": 45, "result_line": 46, "response_saved": true, "usage": { "prompt_tokens": 89, "completion_tokens": 297 }, "estimated_jpy": "0.36015" }, { "model": "unbiased/pareto", "problem_id": "279", "reserve_line": 49, "result_line": 50, "response_saved": true, "usage": { "prompt_tokens": 82, "completion_tokens": 231 }, "estimated_jpy": "0.2848125" }, { "model": "unbiased/pareto", "problem_id": "2832", "reserve_line": 53, "result_line": 54, "response_saved": true, "usage": { "prompt_tokens": 92, "completion_tokens": 262 }, "estimated_jpy": "0.322665" }, { "model": "unbiased/pareto", "problem_id": "3367", "reserve_line": 63, "result_line": 64, "response_saved": true, "usage": { "prompt_tokens": 92, "completion_tokens": 504 }, "estimated_jpy": "0.58947" }, { "model": "unbiased/pareto", "problem_id": "2341", "reserve_line": 67, "result_line": 68, "response_saved": true, "usage": { "prompt_tokens": 55, "completion_tokens": 128 }, "estimated_jpy": "0.1613325" }, { "model": "unbiased/pareto", "problem_id": "1300", "reserve_line": 71, "result_line": 72, "response_saved": true, "usage": { "prompt_tokens": 102, "completion_tokens": 581 }, "estimated_jpy": "0.6780375" }, { "model": "unbiased/pareto", "problem_id": "2416", "reserve_line": 81, "result_line": 82, "response_saved": true, "usage": { "prompt_tokens": 94, "completion_tokens": 626 }, "estimated_jpy": "0.72471" }, { "model": "unbiased/pareto", "problem_id": "2169", "reserve_line": 85, "result_line": 86, "response_saved": true, "usage": { "prompt_tokens": 73, "completion_tokens": 522 }, "estimated_jpy": "0.6023325" }, { "model": "unbiased/pareto", "problem_id": "2422", "reserve_line": 89, "result_line": 90, "response_saved": true, "usage": { "prompt_tokens": 81, "completion_tokens": 1066 }, "estimated_jpy": "1.2050325" }, { "model": "unbiased/pareto", "problem_id": "1203", "reserve_line": 99, "result_line": 100, "response_saved": true, "usage": { "prompt_tokens": 69, "completion_tokens": 735 }, "estimated_jpy": "0.835695" }, { "model": "unbiased/pareto", "problem_id": "1480", "reserve_line": 103, "result_line": 104, "response_saved": true, "usage": { "prompt_tokens": 80, "completion_tokens": 447 }, "estimated_jpy": "0.5222175" }, { "model": "unbiased/pareto", "problem_id": "295", "reserve_line": 107, "result_line": 108, "response_saved": true, "usage": { "prompt_tokens": 74, "completion_tokens": 305 }, "estimated_jpy": "0.3634575" }, { "model": "unbiased/pareto", "problem_id": "3494", "reserve_line": 117, "result_line": 118, "response_saved": true, "usage": { "prompt_tokens": 92, "completion_tokens": 368 }, "estimated_jpy": "0.43953" }, { "model": "unbiased/pareto", "problem_id": "2751", "reserve_line": 121, "result_line": 122, "response_saved": true, "usage": { "prompt_tokens": 61, "completion_tokens": 635 }, "estimated_jpy": "0.722505" }, { "model": "unbiased/pareto", "problem_id": "1392", "reserve_line": 125, "result_line": 126, "response_saved": true, "usage": { "prompt_tokens": 95, "completion_tokens": 531 }, "estimated_jpy": "0.62034" }, { "model": "unbiased/pareto", "problem_id": "2386", "reserve_line": 135, "result_line": null, "response_saved": false, "usage": null, "estimated_jpy": null }, { "model": "unbiased/pareto", "problem_id": "3453", "reserve_line": 140, "result_line": 141, "response_saved": true, "usage": { "prompt_tokens": 69, "completion_tokens": 492 }, "estimated_jpy": "0.5677875" }, { "model": "unbiased/pareto", "problem_id": "2243", "reserve_line": 144, "result_line": 145, "response_saved": true, "usage": { "prompt_tokens": 77, "completion_tokens": 133 }, "estimated_jpy": "0.17493" }, { "model": "unbiased/pareto", "problem_id": "3680", "reserve_line": 154, "result_line": 155, "response_saved": true, "usage": { "prompt_tokens": 71, "completion_tokens": 654 }, "estimated_jpy": "0.7471275" }, { "model": "unbiased/pareto", "problem_id": "3623", "reserve_line": 158, "result_line": 159, "response_saved": true, "usage": { "prompt_tokens": 73, "completion_tokens": 712 }, "estimated_jpy": "0.8118075" }, { "model": "unbiased/pareto", "problem_id": "3434", "reserve_line": 162, "result_line": 163, "response_saved": true, "usage": { "prompt_tokens": 82, "completion_tokens": 324 }, "estimated_jpy": "0.387345" }, { "model": "unbiased/pareto", "problem_id": "3414", "reserve_line": 172, "result_line": 173, "response_saved": true, "usage": { "prompt_tokens": 69, "completion_tokens": 718 }, "estimated_jpy": "0.8169525" }, { "model": "unbiased/pareto", "problem_id": "3326", "reserve_line": 176, "result_line": 177, "response_saved": false, "usage": null, "estimated_jpy": null }, { "model": "unbiased/pareto", "problem_id": "3376", "reserve_line": 183, "result_line": 184, "response_saved": true, "usage": { "prompt_tokens": 70, "completion_tokens": 221 }, "estimated_jpy": "0.2693775" } ] }, "correction": { "date": "2026-09-24", "supersedes_commit": "501d255", "notes": { "withdrawal": { "ja": "訂正：当初の欠測ゼロ補完による主表を撤回します。欠測をモデル誤答として扱うのは不適切でした。Xの「1/10」という費用倍率の誤記も撤回します。予約30件で割った1件価格を削除し、保存usage件数を示します。", "en": "Correction: we withdraw the original main table that imputed missing responses as failures. Missing data must not be treated as model errors. We also withdraw the erroneous “1/10” cost ratio on X. Per-request prices divided by 30 reservations are removed; saved usage counts are shown instead." }, "selection": { "ja": "現行の主表は全3モデル共通で応答が保存された27問だけの参考比較です。2471/2386/3326を全モデルから事後除外しました。計画どおりの成功試験ではなく、順位は断定できません。共通27問にも停止後の続行分が含まれ、条件違反は解消されません。", "en": "The current main table is a reference comparison of the 27 questions with saved responses for all 3 models. IDs 2471/2386/3326 were excluded post hoc from every model. This was not a successful execution of the planned protocol and cannot establish a ranking. The common 27 still include post-halt continuations; filtering does not cure protocol violations." }, "cost": { "ja": "費用不明はunbiased/paretoの2471/2386/3326の3件です。当初summaryは2471のusage未保存を見落とし、2件と誤記しました。2133円は予算予約であり、実費上限の保証ではありません。保存usageの推計と、欠測・上流失敗を含む実費未確定を分離します。", "en": "Costs are unknown for 3 unbiased/pareto rows: 2471/2386/3326. The original summary missed unsaved usage for 2471 and incorrectly reported 2. JPY 2133 was a budget reservation, not a guaranteed spending cap. Saved-usage estimates are separate from unsettled charges including missing usage and upstream failures." }, "halt": { "ja": "測定側不備：usage>4096で停止した後、原因解消の確認・明示的な再開承認なしに4件を続行しました。03:33:54 UTCのconfig revision後、anthropic/claude-opus-5.5/3326 → unbiased/pareto/3376 → anthropic/claude-opus-5.5/3376 → openai/gpt-6-astra/3376です。runnerに恒久haltがなく、実行ごとにhalt状態をクリアしていました。上限を保証できたとは言えません。", "en": "Measurement failure: after the usage>4096 halt, 4 requests continued without confirmation of a resolved cause or explicit restart approval. After the 03:33:54 UTC config revision: anthropic/claude-opus-5.5/3326 → unbiased/pareto/3376 → anthropic/claude-opus-5.5/3376 → openai/gpt-6-astra/3376. The runner lacked a persistent halt and cleared halt state on each execution. We cannot claim the cap was guaranteed." }, "retries": { "ja": "「再試行なし」を訂正：クライアント再試行は0回ですが、gatewayは同一モデルを最大2回試行し得ます。上流失敗のusageは不明で、fallback_depth=0も内部再試行なしの証明ではありません。", "en": "Correction to “no retries”: client retries were 0, but the gateway could make up to 2 attempts for the same model. Usage for failed upstream attempts is unknown; fallback_depth=0 does not prove absence of internal retries." }, "finish": { "ja": "gatewayは非streamのfinish_reasonをstopへ書き換えます。出力打切りは未検証です。servedはgatewayが示した指定model情報であり、上流モデル実体を証明しません。identity判定も途中でserved-model一致のみへ変更されました。", "en": "The gateway rewrites non-stream finish_reason to stop. Output truncation is unverified. served is requested-model information reported by the gateway, not proof of the upstream model identity. The identity rule was also changed mid-run to served-model match only." } }, "rules": [ "Intersect saved response IDs across all three models without consulting pass/fail outcomes.", "Treat unsaved responses as unknown, never as model errors.", "Recompute common-set counts from source per-question judgments, not summary aggregates.", "Recompute each saved usage row with Decimal and ledger-manifest prices; missing usage is null, not zero.", "Preserve all original rows and the withdrawn summary separately; do not invent replacement answers.", "Post-hoc filtering does not restore protocol compliance." ], "old_table_problems": [ "Missing responses zero-imputed as model-performance primary metric.", "2471 omitted from cost-unknown list (2 claimed; 3 actual).", "Per-request /30 costs hid missing usage; withdrawn along with X 1/10 ratio.", "No-retries wording ignored gateway attempts.", "Halt continuation, finish_reason rewriting and upstream identity limits not adequately disclosed." ], "withdrawn_history": { "status": "WITHDRAWN: historical values only; not current conclusions", "summary": { "manifest_fingerprint": "f5cd4e44563189aff73321360b4b0d5f5d9addfc97d4b3b6e0cfc24d20a54a8a", "requests_reserved": 90, "results": 89, "normal_answers_total": 87, "success_rate_over_90": 0.9667, "success_rate_wilson95": [ 0.9065, 0.9886 ], "http_errors": 0, "abandoned": [ [ "unbiased/pareto", "2386" ] ], "reservation_jpy_total": 2133, "usage_estimated_sell_jpy_total": 217.3776, "usage_cost_unknown_rows": [ "unbiased/pareto/3326 (usage_excess, usage not saved; wallet charged per returned usage)", "unbiased/pareto/2386 (reserved, process killed mid-request; gateway completion unknown)" ], "per_model": { "anthropic/claude-opus-5.5": { "strict_prompt_accuracy_primary": 0.8667, "strict_prompt_wilson95": [ 0.7032, 0.9469 ], "strict_instruction_accuracy": 0.9286, "loose_prompt_accuracy": 0.9667, "loose_instruction_accuracy": 0.9821, "normal_answers": 30, "strict_prompt_accuracy_normal_only": 0.8667, "latency_normal_median_s": 11.212, "latency_normal_p90_s": 21.503, "usage_estimated_sell_jpy_total": 108.957, "usage_estimated_sell_jpy_per_request_of_30": 3.6319, "rows_with_usage": 30, "failures": [] }, "openai/gpt-6-astra": { "strict_prompt_accuracy_primary": 0.8333, "strict_prompt_wilson95": [ 0.6644, 0.9266 ], "strict_instruction_accuracy": 0.8929, "loose_prompt_accuracy": 0.8667, "loose_instruction_accuracy": 0.9107, "normal_answers": 30, "strict_prompt_accuracy_normal_only": 0.8333, "latency_normal_median_s": 12.818, "latency_normal_p90_s": 21.907, "usage_estimated_sell_jpy_total": 92.3851, "usage_estimated_sell_jpy_per_request_of_30": 3.0795, "rows_with_usage": 30, "failures": [] }, "unbiased/pareto": { "strict_prompt_accuracy_primary": 0.7333, "strict_prompt_wilson95": [ 0.5555, 0.8582 ], "strict_instruction_accuracy": 0.7857, "loose_prompt_accuracy": 0.8, "loose_instruction_accuracy": 0.8214, "normal_answers": 27, "strict_prompt_accuracy_normal_only": 0.8148, "latency_normal_median_s": 18.42, "latency_normal_p90_s": 40.869, "usage_estimated_sell_jpy_total": 16.0355, "usage_estimated_sell_jpy_per_request_of_30": 0.5345, "rows_with_usage": 27, "failures": [ { "problem_id": "2471", "reason": "content_not_saved_identity_rule_halt_served_matched", "http": 200 }, { "problem_id": "2386", "reason": "reserved_but_result_lost_process_killed", "http": null }, { "problem_id": "3326", "reason": "usage_excess_halt_content_not_saved", "http": 200 } ] } } } } } }