{
 "bench_id": "b003-kessan",
 "slug": "kessan-2026-09-28",
 "title": "決算PDFから数字を拾う（3AI・6課題・各3回）",
 "field": "PDF・決算書の読み取り",
 "field_key": "kessan",
 "round": 1,
 "run_date": "2026-09-28",
 "timezone": "Asia/Tokyo",
 "version": "v1",
 "conductor": "AInformation編集部",
 "funder": "AInformation（自社負担）",
 "sponsor": null,
 "method": {
  "tasks_n": 6,
  "repeats": 3,
  "ais_n": 3,
  "trials_total": 54,
  "trials_valid": 45,
  "excluded": [
   {
    "reason": "our_error",
    "n": 9
   }
  ],
  "new_chat_each_time": true,
  "order_rotated": true,
  "order_by_repeat": {
   "1": [
    "ChatGPT",
    "Gemini",
    "Claude"
   ],
   "2": [
    "Gemini",
    "Claude",
    "ChatGPT"
   ],
   "3": [
    "Claude",
    "ChatGPT",
    "Gemini"
   ]
  },
  "timeout_seconds": 300,
  "scoring": "機械採点のみ（AIに採点させていない）",
  "materials_generated_at": "2026-09-27",
  "materials_note": "架空の株式会社サンプル商事の決算の概要（3ページのPDF・単位は百万円）。数は作るときに決めてある",
  "counted_outcomes": [
   "scored",
   "refused",
   "timeout"
  ],
  "excluded_outcomes": [
   "limit",
   "our_error",
   "human_check"
  ],
  "answer_cleanup": "画面から答えを読み取るとき、AIによっては読み上げ用の1行や考えている途中の要約が答えと同じ枠に入る。採点の前に「答えの先頭にあって、あとでそっくり同じ行がもう一度出てくる」ところまでを落とす。4AIとも同じ処理をする。落とす前の答えは answers/ にそのまま置いてある",
  "csv_url": "/data/research/bench/kessan-2026-09-28/latest.csv"
 },
 "ais": [
  {
   "name": "ChatGPT",
   "plan": "ChatGPT Pro",
   "model_shown": "中程度",
   "checked_at": "2026-09-27 21:16"
  },
  {
   "name": "Gemini",
   "plan": "Google AI Pro",
   "model_shown": "Flash 拡張",
   "checked_at": "2026-09-27 21:16"
  },
  {
   "name": "Claude",
   "plan": "Claude Max",
   "model_shown": "Opus 4.8 高",
   "checked_at": "2026-09-27 21:16"
  }
 ],
 "summary": {
  "headline": "6課題を各3回ずつ投げたところ、正解率が一番高かったのはChatGPT・Claudeの100.0%で同率だった",
  "winner": null,
  "note": "1回の検証の結果。日や課題が変われば順位は変わる"
 },
 "scores": [
  {
   "ai": "ChatGPT",
   "correct": 15,
   "trials": 15,
   "accuracy": 1.0,
   "accuracy_display": 100.0,
   "seconds_median": 17.2,
   "seconds_min": 14.0,
   "seconds_max": 57.5,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.8667,
   "instruction_follow_display": 86.7,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 3,
    "human_check": 0
   }
  },
  {
   "ai": "Gemini",
   "correct": 11,
   "trials": 15,
   "accuracy": 0.7333,
   "accuracy_display": 73.3,
   "seconds_median": 28.6,
   "seconds_min": 20.7,
   "seconds_max": 60.4,
   "consistency": 0.8,
   "consistency_display": 80.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.1667,
   "instruction_follow_display": 16.7,
   "failures": {
    "refused": 3,
    "timeout": 0,
    "limit": 0,
    "our_error": 3,
    "human_check": 0
   }
  },
  {
   "ai": "Claude",
   "correct": 15,
   "trials": 15,
   "accuracy": 1.0,
   "accuracy_display": 100.0,
   "seconds_median": 13.9,
   "seconds_min": 13.8,
   "seconds_max": 15.3,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.6,
   "instruction_follow_display": 60.0,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 3,
    "human_check": 0
   }
  }
 ],
 "tasks": [
  {
   "task_id": "t1",
   "level": "易",
   "prompt": "添付した決算の概要（単位は百万円）から、3つの数字を拾ってください。\n\n決まり\n・答えはちょうど3行。売上高: ◯／営業利益: ◯／当期純利益: ◯ の順で書く\n・数は半角で、資料に出ている単位（百万円）のまま書く\n・計算や補足は書かない",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）売上高＝60500（誤差±0）／営業利益＝3450（誤差±0）／当期純利益＝2220（誤差±0）",
   "materials": [
    "/data/research/bench/kessan-2026-09-28/materials/kessan.pdf"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 57.5,
       "chars": 36,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_chatgpt_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/chatgpt_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.2,
       "chars": 33,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.2,
       "chars": 33,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 30.0,
       "chars": 58,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 33.1,
       "chars": 58,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "refused",
       "seconds": 27.0,
       "chars": 35,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 168,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_claude_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/claude_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.8,
       "chars": 122,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.8,
       "chars": 64,
       "why": "数値すべて一致：売上高 ○（正解60500）／営業利益 ○（正解3450）／当期純利益 ○（正解2220）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t1_claude_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t2",
   "level": "易",
   "prompt": "添付した決算の概要から、期末の従業員数を答えてください。\n\n決まり\n・答えはちょうど1行。従業員数: ◯◯◯名 の形で書く\n・説明は書かない",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）従業員数＝505（誤差±0）",
   "materials": [
    "/data/research/bench/kessan-2026-09-28/materials/kessan.pdf"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.1,
       "chars": 10,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_chatgpt_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/chatgpt_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.1,
       "chars": 10,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.4,
       "chars": 10,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 24.0,
       "chars": 17,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 60.4,
       "chars": 17,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.7,
       "chars": 17,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 37,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_claude_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/claude_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 37,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 37,
       "why": "数値すべて一致：従業員数 ○（正解505）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t2_claude_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t3",
   "level": "中",
   "prompt": "添付した決算の概要から、当年度の営業利益率（営業利益 ÷ 売上高）を出してください。\n\n決まり\n・答えはちょうど1行。営業利益率: ◯.◯% の形で書く\n・小数第1位まで。説明は書かない",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）営業利益率＝5.7（誤差±0.06）",
   "materials": [
    "/data/research/bench/kessan-2026-09-28/materials/kessan.pdf"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.2,
       "chars": 19,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_chatgpt_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/chatgpt_03_t3.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.3,
       "chars": 11,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.6,
       "chars": 19,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.9,
       "chars": 11,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "refused",
       "seconds": 39.3,
       "chars": 60,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.8,
       "chars": 18,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.8,
       "chars": 39,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_claude_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/claude_03_t3.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 137,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.8,
       "chars": 39,
       "why": "数値すべて一致：営業利益率 ○（正解5.7）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t3_claude_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t4",
   "level": "中",
   "prompt": "添付した決算の概要から、当年度の販売費及び一般管理費の内訳を全部書き出してください。\n\n決まり\n・答えはちょうど6行。費目名: 金額 の形で、資料に出てくる順で書く\n・数は半角で、資料の単位（百万円）のまま書く\n・前年度の数は書かない",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）人件費＝8250（誤差±0）／物流費＝5440（誤差±0）／広告宣伝費＝2060（誤差±0）／地代家賃＝1500（誤差±0）／減価償却費＝560（誤差±0）／その他＝940（誤差±0）",
   "materials": [
    "/data/research/bench/kessan-2026-09-28/materials/kessan.pdf"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.4,
       "chars": 66,
       "why": "数値すべて一致：人件費 ○（正解8250）／物流費 ○（正解5440）／広告宣伝費 ○（正解2060）／地代家賃 ○（正解1500）／減価償却費 ○（正解560）／その他 ○（正解940）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t4_chatgpt_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/chatgpt_04_t4.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.2,
       "chars": 62,
       "why": "数値すべて一致：人件費 ○（正解8250）／物流費 ○（正解5440）／広告宣伝費 ○（正解2060）／地代家賃 ○（正解1500）／減価償却費 ○（正解560）／その他 ○（正解940）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t4_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "ChatGPT: 入力欄が無い（ログイン切れ）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 45.5,
       "chars": 79,
       "why": "数値すべて一致：人件費 ○（正解8250）／物流費 ○（正解5440）／広告宣伝費 ○（正解2060）／地代家賃 ○（正解1500）／減価償却費 ○（正解560）／その他 ○（正解940）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t4_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 48.5,
       "chars": 79,
       "why": "数値すべて一致：人件費 ○（正解8250）／物流費 ○（正解5440）／広告宣伝費 ○（正解2060）／地代家賃 ○（正解1500）／減価償却費 ○（正解560）／その他 ○（正解940）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t4_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "Gemini: 入力欄が無い（ログイン切れ）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 127,
       "why": "数値すべて一致：人件費 ○（正解8250）／物流費 ○（正解5440）／広告宣伝費 ○（正解2060）／地代家賃 ○（正解1500）／減価償却費 ○（正解560）／その他 ○（正解940）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t4_claude_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/claude_04_t4.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 15.3,
       "chars": 114,
       "why": "数値すべて一致：人件費 ○（正解8250）／物流費 ○（正解5440）／広告宣伝費 ○（正解2060）／地代家賃 ○（正解1500）／減価償却費 ○（正解560）／その他 ○（正解940）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t4_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "Claude: 入力欄が無い（ログイン切れ）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t5",
   "level": "難",
   "prompt": "添付した決算の概要で、販売費及び一般管理費のうち前年度より金額が増えた費目を全部挙げてください。\n\n決まり\n・1費目につき1行。行の先頭は ・ にする。費目名だけを書く\n・減った費目・変わらなかった費目は書かない",
   "scorer": "set_match",
   "answer_key": "（機械採点 set_match）人件費［人件費］／減価償却費［減価償却費］　作り足しは0件まで",
   "materials": [
    "/data/research/bench/kessan-2026-09-28/materials/kessan.pdf"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.0,
       "chars": 11,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t5_chatgpt_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/chatgpt_05_t5.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.1,
       "chars": 11,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t5_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（ChatGPT: 入力欄が無い（ログイン切れ））",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 27.1,
       "chars": 24,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t5_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "refused",
       "seconds": 23.9,
       "chars": 56,
       "why": "断られた（ポリシー）。不正解として分母に入れる",
       "instruction_follow": null,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t5_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Gemini: 入力欄が無い（ログイン切れ））",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.0,
       "chars": 56,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t5_claude_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/claude_05_t5.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 86,
       "why": "2件すべて一致・作り足し0件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t5_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 入力欄が無い（ログイン切れ））",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t6",
   "level": "難",
   "prompt": "添付した決算の概要から数字を拾い、JSON だけを出力してください。\n\n決まり\n・出力は JSON だけ。前後に説明文を書かない\n・形は {\"sales\": 売上高, \"ec_sales\": ECの売上, \"next_year_forecast\": 翌年度の売上見通し, \"employees\": 期末の従業員数} にする\n・数は数値で入れる（文字列にしない・カンマを入れない）。単位は資料のまま（百万円）",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）売上高＝60500（誤差±0）／EC売上＝20300（誤差±0）／翌年度の売上見通し＝62500（誤差±0）",
   "materials": [
    "/data/research/bench/kessan-2026-09-28/materials/kessan.pdf"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.1,
       "chars": 75,
       "why": "数値すべて一致：売上高 ○（正解60500）／EC売上 ○（正解20300）／翌年度の売上見通し ○（正解62500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t6_chatgpt_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/chatgpt_06_t6.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.2,
       "chars": 75,
       "why": "数値すべて一致：売上高 ○（正解60500）／EC売上 ○（正解20300）／翌年度の売上見通し ○（正解62500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t6_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（ChatGPT: 入力欄が無い（ログイン切れ））",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 42.4,
       "chars": 39,
       "why": "数値が違う：売上高 ×（正解60500・sales の行が無い）／EC売上 ×（正解20300・ec_sales の行が無い）／翌年度の売上見通し ×（正解62500・next_year_forecast の行が無い）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t6_gemini_1.txt",
       "shot": null
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.8,
       "chars": 84,
       "why": "数値すべて一致：売上高 ○（正解60500）／EC売上 ○（正解20300）／翌年度の売上見通し ○（正解62500）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t6_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Gemini: 入力欄が無い（ログイン切れ））",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 179,
       "why": "数値すべて一致：売上高 ○（正解60500）／EC売上 ○（正解20300）／翌年度の売上見通し ○（正解62500）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t6_claude_1.txt",
       "shot": "/data/research/bench/kessan-2026-09-28/shots/claude_06_t6.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 179,
       "why": "数値すべて一致：売上高 ○（正解60500）／EC売上 ○（正解20300）／翌年度の売上見通し ○（正解62500）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/kessan-2026-09-28/answers/t6_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": null,
       "outcome": "our_error",
       "seconds": null,
       "chars": 0,
       "why": "先に止めた（Claude: 入力欄が無い（ログイン切れ））",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    }
   ]
  }
 ],
 "limits": [
  "1日1回の測定である。日や課題が変われば順位は変わる",
  "プランは各AIの有料版（ChatGPT Pro／Google AI Pro／Claude Max／Microsoft 365 Personal）。無料版の結果ではない",
  "既定のモデル・既定のモードだけで測っている（Deep Research などは使っていない）",
  "課題は架空の株式会社サンプル商事の自作データ。実在の企業のデータでの結果ではない",
  "機械で採点できる課題だけを扱っている。文章のうまさ・読みやすさは測っていない",
  "ブラウザで人が使うのと同じ画面から測っている。APIの結果ではない",
  "画面に出ているモデル名を読み取れないAIがある回は、その欄を空（null）にしている。読み取れた回はそのまま載せている"
 ],
 "license": "CC BY 4.0",
 "attribution": "出典として AInformation調べ と、使ったページのURL・測定日を書いてください。",
 "page_url": "https://ainformation.jp/research/bench/kessan-2026-09-28"
}