{
 "bench_id": "b004-shirabemono",
 "slug": "shirabemono-2026-09-29",
 "title": "答えが一意に決まる事実を調べる（4AI・6課題・各3回）",
 "field": "調べもの・事実確認",
 "field_key": "shirabemono",
 "round": 1,
 "run_date": "2026-09-29",
 "timezone": "Asia/Tokyo",
 "version": "v1",
 "conductor": "AInformation編集部",
 "funder": "AInformation（自社負担）",
 "sponsor": null,
 "method": {
  "tasks_n": 6,
  "repeats": 3,
  "ais_n": 4,
  "trials_total": 72,
  "trials_valid": 54,
  "excluded": [
   {
    "reason": "human_check",
    "n": 18
   }
  ],
  "new_chat_each_time": true,
  "order_rotated": true,
  "order_by_repeat": {
   "1": [
    "ChatGPT",
    "Gemini",
    "Claude",
    "Copilot"
   ],
   "2": [
    "Gemini",
    "Claude",
    "Copilot",
    "ChatGPT"
   ],
   "3": [
    "Claude",
    "Copilot",
    "ChatGPT",
    "Gemini"
   ]
  },
  "timeout_seconds": 300,
  "scoring": "機械採点のみ（AIに採点させていない）",
  "materials_generated_at": "2026-09-28",
  "materials_note": "答えが計算で一意に決まる問い5つ（曜日・うるう年・単位換算・日数・約数）。実在の企業・人物の話は使わない",
  "counted_outcomes": [
   "scored",
   "refused",
   "timeout"
  ],
  "excluded_outcomes": [
   "limit",
   "our_error",
   "human_check"
  ],
  "answer_cleanup": "画面から答えを読み取るとき、AIによっては読み上げ用の1行や考えている途中の要約が答えと同じ枠に入る。採点の前に「答えの先頭にあって、あとでそっくり同じ行がもう一度出てくる」ところまでを落とす。4AIとも同じ処理をする。落とす前の答えは answers/ にそのまま置いてある",
  "csv_url": "/data/research/bench/shirabemono-2026-09-29/latest.csv"
 },
 "ais": [
  {
   "name": "ChatGPT",
   "plan": "ChatGPT Pro",
   "model_shown": "中程度",
   "checked_at": "2026-09-28 19:51"
  },
  {
   "name": "Gemini",
   "plan": "Google AI Pro",
   "model_shown": "Flash 拡張",
   "checked_at": "2026-09-28 19:51"
  },
  {
   "name": "Claude",
   "plan": "Claude Max",
   "model_shown": null,
   "checked_at": "2026-09-28 19:51"
  },
  {
   "name": "Copilot",
   "plan": "Microsoft 365 Personal",
   "model_shown": null,
   "checked_at": "2026-09-28 19:51"
  }
 ],
 "summary": {
  "headline": "6課題を各3回ずつ投げたところ、正解率が一番高かったのはChatGPTで18/18試行の100.0%だった（2位はCopilotの72.2%）",
  "winner": "ChatGPT",
  "note": "1回の検証の結果。日や課題が変われば順位は変わる"
 },
 "scores": [
  {
   "ai": "ChatGPT",
   "correct": 18,
   "trials": 18,
   "accuracy": 1.0,
   "accuracy_display": 100.0,
   "seconds_median": 20.2,
   "seconds_min": 14.0,
   "seconds_max": 26.7,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": 1.0,
   "source_alive_display": 100.0,
   "instruction_follow": 1.0,
   "instruction_follow_display": 100.0,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Gemini",
   "correct": 4,
   "trials": 18,
   "accuracy": 0.2222,
   "accuracy_display": 22.2,
   "seconds_median": 33.0,
   "seconds_min": 23.6,
   "seconds_max": 78.6,
   "consistency": 0.3333,
   "consistency_display": 33.3,
   "source_alive": 0.8333,
   "source_alive_display": 83.3,
   "instruction_follow": 0.5,
   "instruction_follow_display": 50.0,
   "failures": {
    "refused": 8,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Claude",
   "correct": 0,
   "trials": 0,
   "accuracy": null,
   "accuracy_display": null,
   "seconds_median": null,
   "seconds_min": null,
   "seconds_max": null,
   "consistency": null,
   "consistency_display": null,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": null,
   "instruction_follow_display": null,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 18
   }
  },
  {
   "ai": "Copilot",
   "correct": 13,
   "trials": 18,
   "accuracy": 0.7222,
   "accuracy_display": 72.2,
   "seconds_median": 20.1,
   "seconds_min": 13.9,
   "seconds_max": 26.4,
   "consistency": 0.6667,
   "consistency_display": 66.7,
   "source_alive": 0.9333,
   "source_alive_display": 93.3,
   "instruction_follow": 0.3125,
   "instruction_follow_display": 31.2,
   "failures": {
    "refused": 0,
    "timeout": 2,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  }
 ],
 "tasks": [
  {
   "task_id": "t1",
   "level": "易",
   "prompt": "次の2つの問いに答えてください。\n\nf1. 2027年4月30日は何曜日ですか。\nf2. 西暦2038年はうるう年ですか。「うるう年です」か「うるう年ではありません」で答えてください。\n\n決まり\n・答えはちょうど2行。f1: ◯／f2: ◯ の形で書く\n・理由と計算の過程は書かない",
   "scorer": "contains_all",
   "answer_key": "（機械採点 contains_all）f1＝金曜または金曜日／f2＝うるう年ではありませんまたはうるう年でないまたは平年またはいいえ",
   "materials": [
    "/data/research/bench/shirabemono-2026-09-29/materials/facts_q12.txt"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 23,
       "why": "2項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_chatgpt_1.txt",
       "shot": "/data/research/bench/shirabemono-2026-09-29/shots/chatgpt_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.1,
       "chars": 23,
       "why": "2項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.0,
       "chars": 23,
       "why": "2項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "refused",
       "seconds": 29.9,
       "chars": 60,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "refused",
       "seconds": 26.8,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "refused",
       "seconds": 23.7,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 2,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 14.0,
       "chars": 33,
       "why": "2項目中0項目あり・足りない f1／f2",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.0,
       "chars": 23,
       "why": "2項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 23,
       "why": "2項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t1_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t2",
   "level": "易",
   "prompt": "次の3つの問いに答えてください。\n\nf3. 13マイルは何キロメートルですか。1マイル＝1.609344キロメートルとして、小数第2位まで四捨五入して答えてください。\nf4. 2024年7月23日 から 2026年9月29日 までは何日ありますか（両端の差の日数）。\nf5. 259 の正の約数は全部でいくつありますか。\n\n決まり\n・答えはちょうど3行。f3: ◯／f4: ◯／f5: ◯ の形で書く\n・数は半角で書く。単位は書かなくてよい。計算の過程は書かない",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）f3＝20.92（誤差±0.011）／f4＝798（誤差±0）／f5＝4（誤差±0）",
   "materials": [
    "/data/research/bench/shirabemono-2026-09-29/materials/facts_q345.txt"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.3,
       "chars": 23,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_chatgpt_1.txt",
       "shot": "/data/research/bench/shirabemono-2026-09-29/shots/chatgpt_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.0,
       "chars": 23,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 23,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 23.6,
       "chars": 39,
       "why": "数値が違う：f3 ×（正解20.92・f3 の行が無い）／f4 ×（正解798・f4 の行が無い）／f5 ×（正解4・f5 の行が無い）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.7,
       "chars": 23,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 33.1,
       "chars": 23,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 2,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 23,
       "why": "数値が違う：f3 ○（正解20.92）／f4 ×（正解798・答え3.0）／f5 ○（正解4）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 23,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 23,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t2_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t3",
   "level": "中",
   "prompt": "次の5つの問いに答えてください。\n\nf1. 2027年4月30日は何曜日ですか。\nf2. 西暦2038年はうるう年ですか。「うるう年です」か「うるう年ではありません」で答えてください。\nf3. 13マイルは何キロメートルですか。1マイル＝1.609344キロメートルとして、小数第2位まで四捨五入して答えてください。\nf4. 2024年7月23日 から 2026年9月29日 までは何日ありますか（両端の差の日数）。\nf5. 259 の正の約数は全部でいくつありますか。\n\n決まり\n・答えはちょうど5行。f1: ◯ から f5: ◯ まで、この順で書く\n・数は半角で書く。計算の過程と理由は書かない",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）f3＝20.92（誤差±0.011）／f4＝798（誤差±0）／f5＝4（誤差±0）",
   "materials": [
    "/data/research/bench/shirabemono-2026-09-29/materials/facts_q.txt"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.3,
       "chars": 47,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_chatgpt_1.txt",
       "shot": "/data/research/bench/shirabemono-2026-09-29/shots/chatgpt_03_t3.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.1,
       "chars": 47,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.3,
       "chars": 47,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "refused",
       "seconds": 42.2,
       "chars": 60,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 35.9,
       "chars": 26,
       "why": "数値が違う：f3 ×（正解20.92・f3 の行が無い）／f4 ×（正解798・f4 の行が無い）／f5 ×（正解4・f5 の行が無い）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "refused",
       "seconds": 20.7,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 2,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.2,
       "chars": 47,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.4,
       "chars": 47,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t3_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "timeout",
       "seconds": 301.2,
       "chars": 0,
       "why": "時間切れ（制限時間まで答えが止まらなかった）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t4",
   "level": "中",
   "prompt": "1マイルが何メートルかを確かめられる公式の資料を3つ挙げてください。\n\n決まり\n・答えはちょうど3行。1行につきURLを1本だけ書く\n・いま開けるURLだけを書く。存在しないURLは書かない\n・説明は書かない",
   "scorer": "url_alive",
   "answer_key": "（機械採点 url_alive）挙げたURLを実際に開いて実在を確かめる。",
   "materials": [],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 263,
       "why": "URL 3/3本が実在",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_chatgpt_1.txt",
       "shot": "/data/research/bench/shirabemono-2026-09-29/shots/chatgpt_04_t4.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 261,
       "why": "URL 3/3本が実在",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 310,
       "why": "URL 3/3本が実在",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 32.9,
       "chars": 27,
       "why": "URLが0本（3本を求めた）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 23.7,
       "chars": 26,
       "why": "URLが0本（3本を求めた）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "refused",
       "seconds": 29.9,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 2,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.2,
       "chars": 185,
       "why": "URL 3/3本が実在",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 20.0,
       "chars": 231,
       "why": "URL 2/3本が実在・開けない https://www.bipm.org/en/publications/si-brochure/edition-9/a",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 157,
       "why": "URL 3/3本が実在",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t4_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t5",
   "level": "難",
   "prompt": "13マイルは何キロメートルですか。1マイル＝1.609344キロメートルとして、小数第2位まで四捨五入して答えてください。\n\nあわせて、その換算の根拠になる資料のURLを3本挙げてください。\n\n決まり\n・1行目に 答え: ◯◯.◯◯ と書く（数は半角・小数第2位まで）\n・2行目から、1行につきURLを1本だけ、3行書く\n・いま開けるURLだけを書く。存在しないURLは書かない",
   "scorer": "url_alive",
   "answer_key": "（機械採点 url_alive）挙げたURLを実際に開いて実在を確かめる。マイル換算＝20.92",
   "materials": [
    "/data/research/bench/shirabemono-2026-09-29/materials/facts_q3.txt"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.5,
       "chars": 301,
       "why": "URL 3/3本が実在／マイル換算 ○（正解20.92)",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_chatgpt_1.txt",
       "shot": "/data/research/bench/shirabemono-2026-09-29/shots/chatgpt_05_t5.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 320,
       "why": "URL 3/3本が実在／マイル換算 ○（正解20.92)",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.7,
       "chars": 264,
       "why": "URL 3/3本が実在／マイル換算 ○（正解20.92)",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 36.1,
       "chars": 155,
       "why": "URL 2/3本が実在・開けない https://www.britannica.com/science/mile／マイル換算 ○（正解20.92)",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "refused",
       "seconds": 29.8,
       "chars": 56,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 78.6,
       "chars": 161,
       "why": "URL 3/3本が実在／マイル換算 ○（正解20.92)",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 2,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 170,
       "why": "URL 3/3本が実在／マイル換算 ○（正解20.92)",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "timeout",
       "seconds": 300.8,
       "chars": 0,
       "why": "時間切れ（制限時間まで答えが止まらなかった）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 170,
       "why": "URL 3/3本が実在／マイル換算 ○（正解20.92)",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t5_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t6",
   "level": "難",
   "prompt": "次の5つの問いに答え、JSON だけを出力してください。\n\nf1. 2027年4月30日は何曜日ですか。\nf2. 西暦2038年はうるう年ですか。「うるう年です」か「うるう年ではありません」で答えてください。\nf3. 13マイルは何キロメートルですか。1マイル＝1.609344キロメートルとして、小数第2位まで四捨五入して答えてください。\nf4. 2024年7月23日 から 2026年9月29日 までは何日ありますか（両端の差の日数）。\nf5. 259 の正の約数は全部でいくつありますか。\n\n決まり\n・出力は JSON だけ。前後に説明文を書かない\n・形は {\"f1\": \"…\", \"f2\": \"…\", \"f3\": 数値, \"f4\": 数値, \"f5\": 数値} にする\n・f3・f4・f5 は数値で入れる（文字列にしない・単位を付けない）",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）f3＝20.92（誤差±0.011）／f4＝798（誤差±0）／f5＝4（誤差±0）",
   "materials": [
    "/data/research/bench/shirabemono-2026-09-29/materials/facts_q.txt"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.2,
       "chars": 58,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_chatgpt_1.txt",
       "shot": "/data/research/bench/shirabemono-2026-09-29/shots/chatgpt_06_t6.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.3,
       "chars": 58,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.2,
       "chars": 58,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 60.6,
       "chars": 69,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "refused",
       "seconds": 23.6,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 29.9,
       "chars": 39,
       "why": "数値が違う：f3 ×（正解20.92・f3 の行が無い）／f4 ×（正解798・f4 の行が無い）／f5 ×（正解4・f5 の行が無い）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 2,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "human_check",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 人間確認が出た（title に「just a moment」）。押さずに止めた）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.1,
       "chars": 58,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.2,
       "chars": 58,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 58,
       "why": "数値すべて一致：f3 ○（正解20.92）／f4 ○（正解798）／f5 ○（正解4）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/shirabemono-2026-09-29/answers/t6_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  }
 ],
 "limits": [
  "1日1回の測定である。日や課題が変われば順位は変わる",
  "プランは各AIの有料版（ChatGPT Pro／Google AI Pro／Claude Max／Microsoft 365 Personal）。無料版の結果ではない",
  "既定のモデル・既定のモードだけで測っている（Deep Research などは使っていない）",
  "課題は架空の株式会社サンプル商事の自作データ。実在の企業のデータでの結果ではない",
  "機械で採点できる課題だけを扱っている。文章のうまさ・読みやすさは測っていない",
  "ブラウザで人が使うのと同じ画面から測っている。APIの結果ではない",
  "画面に出ているモデル名を読み取れないAIがある回は、その欄を空（null）にしている。読み取れた回はそのまま載せている",
  "この回はClaudeを測れていない（測定中にログインが切れたため）。表のClaudeの欄は試行0で、ほかのAIと比べられる数字ではない"
 ],
 "license": "CC BY 4.0",
 "attribution": "出典として AInformation調べ と、使ったページのURL・測定日を書いてください。",
 "page_url": "https://ainformation.jp/research/bench/shirabemono-2026-09-29"
}