{
 "bench_id": "b006-hyokeisan",
 "slug": "hyokeisan-2026-10-02",
 "title": "売上表から合計と伸び率を出す（4AI・6課題・各3回）",
 "field": "表計算・集計",
 "field_key": "hyokeisan",
 "round": 1,
 "run_date": "2026-10-02",
 "timezone": "Asia/Tokyo",
 "version": "v1",
 "conductor": "AInformation編集部",
 "funder": "AInformation（自社負担）",
 "sponsor": null,
 "method": {
  "tasks_n": 6,
  "repeats": 3,
  "ais_n": 4,
  "trials_total": 72,
  "trials_valid": 72,
  "excluded": [],
  "new_chat_each_time": true,
  "order_rotated": true,
  "order_by_repeat": {
   "1": [
    "ChatGPT",
    "Gemini",
    "Claude",
    "Copilot"
   ],
   "2": [
    "Gemini",
    "Claude",
    "Copilot",
    "ChatGPT"
   ],
   "3": [
    "Claude",
    "Copilot",
    "ChatGPT",
    "Gemini"
   ]
  },
  "timeout_seconds": 300,
  "scoring": "機械採点のみ（AIに採点させていない）",
  "materials_generated_at": "2026-10-01",
  "materials_note": "架空の株式会社サンプル商事の12か月×商品5つの売上（千円）。数は式で決めているので答えが計算で決まる",
  "counted_outcomes": [
   "scored",
   "refused",
   "timeout"
  ],
  "excluded_outcomes": [
   "limit",
   "our_error",
   "human_check"
  ],
  "answer_cleanup": "画面から答えを読み取るとき、AIによっては読み上げ用の1行や考えている途中の要約が答えと同じ枠に入る。採点の前に「答えの先頭にあって、あとでそっくり同じ行がもう一度出てくる」ところまでを落とす。4AIとも同じ処理をする。落とす前の答えは answers/ にそのまま置いてある",
  "csv_url": "/data/research/bench/hyokeisan-2026-10-02/latest.csv"
 },
 "ais": [
  {
   "name": "ChatGPT",
   "plan": "ChatGPT Pro",
   "model_shown": "Medium",
   "checked_at": "2026-10-01 19:59"
  },
  {
   "name": "Gemini",
   "plan": "Google AI Pro",
   "model_shown": "Flash",
   "checked_at": "2026-10-01 19:59"
  },
  {
   "name": "Claude",
   "plan": "Claude Max",
   "model_shown": "Opus 4.8 高",
   "checked_at": "2026-10-01 19:59"
  },
  {
   "name": "Copilot",
   "plan": "Microsoft 365 Personal",
   "model_shown": null,
   "checked_at": "2026-10-01 19:59"
  }
 ],
 "summary": {
  "headline": "6課題を各3回ずつ投げたところ、正解率が一番高かったのはChatGPT・Copilotの100.0%で同率だった",
  "winner": null,
  "note": "1回の検証の結果。日や課題が変われば順位は変わる"
 },
 "scores": [
  {
   "ai": "ChatGPT",
   "correct": 18,
   "trials": 18,
   "accuracy": 1.0,
   "accuracy_display": 100.0,
   "seconds_median": 20.4,
   "seconds_min": 14.1,
   "seconds_max": 29.8,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 1.0,
   "instruction_follow_display": 100.0,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Gemini",
   "correct": 9,
   "trials": 18,
   "accuracy": 0.5,
   "accuracy_display": 50.0,
   "seconds_median": 36.1,
   "seconds_min": 23.7,
   "seconds_max": 72.8,
   "consistency": 0.8333,
   "consistency_display": 83.3,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.9167,
   "instruction_follow_display": 91.7,
   "failures": {
    "refused": 6,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Claude",
   "correct": 15,
   "trials": 18,
   "accuracy": 0.8333,
   "accuracy_display": 83.3,
   "seconds_median": 26.6,
   "seconds_min": 13.9,
   "seconds_max": 44.8,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.1667,
   "instruction_follow_display": 16.7,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Copilot",
   "correct": 18,
   "trials": 18,
   "accuracy": 1.0,
   "accuracy_display": 100.0,
   "seconds_median": 17.1,
   "seconds_min": 13.9,
   "seconds_max": 23.4,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.8889,
   "instruction_follow_display": 88.9,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  }
 ],
 "tasks": [
  {
   "task_id": "t1",
   "level": "易",
   "prompt": "次の売上データ（単位は千円）で、12か月・5商品を全部足した年間の合計はいくつですか。\n\n決まり\n・答えはちょうど1行。合計: ◯◯◯千円 の形で書く\n・数は半角で書く。計算の途中式や説明は書かない\n\n---- 売上データ ここから ----\n月,商品A（タンブラー）,商品B（エコバッグ）,商品C（冷感タオル）,商品D（弁当箱）,商品E（保温ボトル）,合計\n2025-01,9900,14000,4800,12600,13700,55000\n2025-02,10000,13900,4800,11800,13700,54200\n2025-03,10200,12300,7200,9500,15100,54300\n2025-04,10400,11300,10800,7100,13700,53300\n2025-05,10600,10800,15600,5500,13700,56200\n2025-06,10700,14000,21600,4000,13700,64000\n2025-07,10900,13000,28800,3200,13700,69600\n2025-08,11100,13800,31200,3200,13700,73000\n2025-09,11300,10200,20400,4700,13700,60300\n2025-10,11400,11000,9600,7900,13700,53600\n2025-11,11600,11800,6000,11100,13700,54200\n2025-12,11800,11600,4800,13400,15100,56700\n---- 売上データ ここまで ----",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）年間の合計＝704400（誤差±0）",
   "materials": [
    "/data/research/bench/hyokeisan-2026-10-02/materials/uriage.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.7,
       "chars": 12,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_chatgpt_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.4,
       "chars": 12,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.4,
       "chars": 12,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 38.9,
       "chars": 26,
       "why": "数値が違う：年間の合計 ×（正解704400・合計 の行が無い）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.7,
       "chars": 12,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 39.1,
       "chars": 27,
       "why": "数値が違う：年間の合計 ×（正解704400・合計 の行が無い）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 35.5,
       "chars": 73,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_claude_1.txt",
       "shot": "/data/research/bench/hyokeisan-2026-10-02/shots/claude_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 32.5,
       "chars": 161,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.1,
       "chars": 75,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.2,
       "chars": 12,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.2,
       "chars": 12,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 12,
       "why": "数値すべて一致：年間の合計 ○（正解704400）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t1_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t2",
   "level": "易",
   "prompt": "次の売上データ（単位は千円）で、年間の合計が一番大きい商品はどれですか。\n\n決まり\n・答えはちょうど1行。商品名だけを書く\n・理由と計算の説明は書かない\n\n---- 売上データ ここから ----\n月,商品A（タンブラー）,商品B（エコバッグ）,商品C（冷感タオル）,商品D（弁当箱）,商品E（保温ボトル）,合計\n2025-01,9900,14000,4800,12600,13700,55000\n2025-02,10000,13900,4800,11800,13700,54200\n2025-03,10200,12300,7200,9500,15100,54300\n2025-04,10400,11300,10800,7100,13700,53300\n2025-05,10600,10800,15600,5500,13700,56200\n2025-06,10700,14000,21600,4000,13700,64000\n2025-07,10900,13000,28800,3200,13700,69600\n2025-08,11100,13800,31200,3200,13700,73000\n2025-09,11300,10200,20400,4700,13700,60300\n2025-10,11400,11000,9600,7900,13700,53600\n2025-11,11600,11800,6000,11100,13700,54200\n2025-12,11800,11600,4800,13400,15100,56700\n---- 売上データ ここまで ----",
   "scorer": "contains_all",
   "answer_key": "（機械採点 contains_all）一番売れた商品＝商品Eまたは商品E（保温ボトル）",
   "materials": [
    "/data/research/bench/hyokeisan-2026-10-02/materials/uriage.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.2,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_chatgpt_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.5,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.7,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "refused",
       "seconds": 36.0,
       "chars": 63,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 72.8,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 36.4,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.3,
       "chars": 69,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_claude_1.txt",
       "shot": "/data/research/bench/hyokeisan-2026-10-02/shots/claude_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.2,
       "chars": 77,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 27.0,
       "chars": 79,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.4,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t2_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t3",
   "level": "中",
   "prompt": "次の売上データ（単位は千円）で、12月の売上を1月の売上で割った値が一番大きい商品はどれですか。\n\n決まり\n・答えはちょうど1行。商品名だけを書く\n・年間の合計ではなく、12月÷1月 で比べる\n\n---- 売上データ ここから ----\n月,商品A（タンブラー）,商品B（エコバッグ）,商品C（冷感タオル）,商品D（弁当箱）,商品E（保温ボトル）,合計\n2025-01,9900,14000,4800,12600,13700,55000\n2025-02,10000,13900,4800,11800,13700,54200\n2025-03,10200,12300,7200,9500,15100,54300\n2025-04,10400,11300,10800,7100,13700,53300\n2025-05,10600,10800,15600,5500,13700,56200\n2025-06,10700,14000,21600,4000,13700,64000\n2025-07,10900,13000,28800,3200,13700,69600\n2025-08,11100,13800,31200,3200,13700,73000\n2025-09,11300,10200,20400,4700,13700,60300\n2025-10,11400,11000,9600,7900,13700,53600\n2025-11,11600,11800,6000,11100,13700,54200\n2025-12,11800,11600,4800,13400,15100,56700\n---- 売上データ ここまで ----",
   "scorer": "contains_all",
   "answer_key": "（機械採点 contains_all）12月÷1月の伸びが一番大きい商品＝商品Aまたは商品A（タンブラー）",
   "materials": [
    "/data/research/bench/hyokeisan-2026-10-02/materials/uriage.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.2,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_chatgpt_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.1,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.3,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "refused",
       "seconds": 35.9,
       "chars": 56,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 35.8,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 27.0,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.1,
       "chars": 63,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_claude_1.txt",
       "shot": "/data/research/bench/hyokeisan-2026-10-02/shots/claude_03_t3.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 79,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.1,
       "chars": 65,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.0,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.3,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.4,
       "chars": 10,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t3_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t4",
   "level": "中",
   "prompt": "次の売上データ（単位は千円）を、四半期ごとの合計（5商品を足したもの）にしてください。\n\n決まり\n・答えはちょうど4行。Q1: ◯◯◯ / Q2: ◯◯◯ / Q3: ◯◯◯ / Q4: ◯◯◯ の順で書く\n・Q1は1〜3月、Q2は4〜6月、Q3は7〜9月、Q4は10〜12月\n・数は半角で書く。説明は書かない\n\n---- 売上データ ここから ----\n月,商品A（タンブラー）,商品B（エコバッグ）,商品C（冷感タオル）,商品D（弁当箱）,商品E（保温ボトル）,合計\n2025-01,9900,14000,4800,12600,13700,55000\n2025-02,10000,13900,4800,11800,13700,54200\n2025-03,10200,12300,7200,9500,15100,54300\n2025-04,10400,11300,10800,7100,13700,53300\n2025-05,10600,10800,15600,5500,13700,56200\n2025-06,10700,14000,21600,4000,13700,64000\n2025-07,10900,13000,28800,3200,13700,69600\n2025-08,11100,13800,31200,3200,13700,73000\n2025-09,11300,10200,20400,4700,13700,60300\n2025-10,11400,11000,9600,7900,13700,53600\n2025-11,11600,11800,6000,11100,13700,54200\n2025-12,11800,11600,4800,13400,15100,56700\n---- 売上データ ここまで ----",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）Q1＝163500（誤差±0）／Q2＝173500（誤差±0）／Q3＝202900（誤差±0）／Q4＝164500（誤差±0）",
   "materials": [
    "/data/research/bench/hyokeisan-2026-10-02/materials/uriage.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.4,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_chatgpt_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.4,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.6,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.8,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.8,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.7,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 29.3,
       "chars": 118,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_claude_1.txt",
       "shot": "/data/research/bench/hyokeisan-2026-10-02/shots/claude_04_t4.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.3,
       "chars": 100,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 102,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.3,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.1,
       "chars": 43,
       "why": "数値すべて一致：Q1 ○（正解163500）／Q2 ○（正解173500）／Q3 ○（正解202900）／Q4 ○（正解164500）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t4_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t5",
   "level": "難",
   "prompt": "次の売上データ（単位は千円）で、下期（7〜12月）の合計が上期（1〜6月）の合計より大きい商品を全部挙げてください。\n\n決まり\n・1商品につき1行。行の先頭は ・ にする。商品名だけを書く\n・当てはまらない商品は書かない\n・同じ場合（まったく同額）は書かない\n\n---- 売上データ ここから ----\n月,商品A（タンブラー）,商品B（エコバッグ）,商品C（冷感タオル）,商品D（弁当箱）,商品E（保温ボトル）,合計\n2025-01,9900,14000,4800,12600,13700,55000\n2025-02,10000,13900,4800,11800,13700,54200\n2025-03,10200,12300,7200,9500,15100,54300\n2025-04,10400,11300,10800,7100,13700,53300\n2025-05,10600,10800,15600,5500,13700,56200\n2025-06,10700,14000,21600,4000,13700,64000\n2025-07,10900,13000,28800,3200,13700,69600\n2025-08,11100,13800,31200,3200,13700,73000\n2025-09,11300,10200,20400,4700,13700,60300\n2025-10,11400,11000,9600,7900,13700,53600\n2025-11,11600,11800,6000,11100,13700,54200\n2025-12,11800,11600,4800,13400,15100,56700\n---- 売上データ ここまで ----",
   "scorer": "set_match",
   "answer_key": "（機械採点 set_match）商品A［商品A］／商品C［商品C］　作り足しは0件まで",
   "materials": [
    "/data/research/bench/hyokeisan-2026-10-02/materials/uriage.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.8,
       "chars": 23,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_chatgpt_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.4,
       "chars": 23,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.5,
       "chars": 23,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "refused",
       "seconds": 29.9,
       "chars": 54,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 45.2,
       "chars": 39,
       "why": "2件中0件一致・漏れ 商品A／商品C・作り足し1件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "refused",
       "seconds": 24.7,
       "chars": 54,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 26.3,
       "chars": 193,
       "why": "2件中2件一致・作り足し1件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_claude_1.txt",
       "shot": "/data/research/bench/hyokeisan-2026-10-02/shots/claude_05_t5.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 29.3,
       "chars": 81,
       "why": "2件中2件一致・作り足し1件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 29.4,
       "chars": 164,
       "why": "2件中2件一致・作り足し1件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.0,
       "chars": 33,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.0,
       "chars": 33,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 18.0,
       "chars": 24,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t5_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t6",
   "level": "難",
   "prompt": "次の売上データ（単位は千円）を集計し、JSON だけを出力してください。\n\n決まり\n・出力は JSON だけ。前後に説明文を書かない\n・形は {\"total\": 年間の合計, \"top_product\": \"年間の合計が一番大きい商品名\", \"best_month\": \"合計が一番大きい月（YYYY-MM）\", \"best_month_total\": その月の合計} にする\n・数は数値で入れる（文字列にしない・カンマを入れない）\n\n---- 売上データ ここから ----\n月,商品A（タンブラー）,商品B（エコバッグ）,商品C（冷感タオル）,商品D（弁当箱）,商品E（保温ボトル）,合計\n2025-01,9900,14000,4800,12600,13700,55000\n2025-02,10000,13900,4800,11800,13700,54200\n2025-03,10200,12300,7200,9500,15100,54300\n2025-04,10400,11300,10800,7100,13700,53300\n2025-05,10600,10800,15600,5500,13700,56200\n2025-06,10700,14000,21600,4000,13700,64000\n2025-07,10900,13000,28800,3200,13700,69600\n2025-08,11100,13800,31200,3200,13700,73000\n2025-09,11300,10200,20400,4700,13700,60300\n2025-10,11400,11000,9600,7900,13700,53600\n2025-11,11600,11800,6000,11100,13700,54200\n2025-12,11800,11600,4800,13400,15100,56700\n---- 売上データ ここまで ----",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）年間の合計＝704400（誤差±0）／一番大きい月の合計＝73000（誤差±0）",
   "materials": [
    "/data/research/bench/hyokeisan-2026-10-02/materials/uriage.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.6,
       "chars": 91,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_chatgpt_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.5,
       "chars": 91,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 29.8,
       "chars": 91,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "refused",
       "seconds": 35.9,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 48.6,
       "chars": 100,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "refused",
       "seconds": 23.6,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 44.8,
       "chars": 337,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_claude_1.txt",
       "shot": "/data/research/bench/hyokeisan-2026-10-02/shots/claude_06_t6.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 44.6,
       "chars": 233,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 38.8,
       "chars": 323,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.2,
       "chars": 91,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_copilot_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 91,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.4,
       "chars": 91,
       "why": "数値すべて一致：年間の合計 ○（正解704400）／一番大きい月の合計 ○（正解73000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/hyokeisan-2026-10-02/answers/t6_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  }
 ],
 "limits": [
  "1日1回の測定である。日や課題が変われば順位は変わる",
  "プランは各AIの有料版（ChatGPT Pro／Google AI Pro／Claude Max／Microsoft 365 Personal）。無料版の結果ではない",
  "既定のモデル・既定のモードだけで測っている（Deep Research などは使っていない）",
  "課題は架空の株式会社サンプル商事の自作データ。実在の企業のデータでの結果ではない",
  "機械で採点できる課題だけを扱っている。文章のうまさ・読みやすさは測っていない",
  "ブラウザで人が使うのと同じ画面から測っている。APIの結果ではない",
  "画面に出ているモデル名を読み取れないAIがある回は、その欄を空（null）にしている。読み取れた回はそのまま載せている"
 ],
 "license": "CC BY 4.0",
 "attribution": "出典として AInformation調べ と、使ったページのURL・測定日を書いてください。",
 "page_url": "https://ainformation.jp/research/bench/hyokeisan-2026-10-02"
}