{
 "bench_id": "b010-meibo",
 "slug": "meibo-2026-10-19",
 "title": "取引先名の表記ゆれをそろえる（4AI・6課題・各3回）",
 "field": "データ整形・名寄せ",
 "field_key": "meibo",
 "round": 1,
 "run_date": "2026-10-19",
 "timezone": "Asia/Tokyo",
 "version": "v1",
 "conductor": "AInformation編集部",
 "funder": "AInformation（自社負担）",
 "sponsor": null,
 "method": {
  "tasks_n": 6,
  "repeats": 3,
  "ais_n": 4,
  "trials_total": 72,
  "trials_valid": 72,
  "excluded": [],
  "new_chat_each_time": true,
  "order_rotated": true,
  "order_by_repeat": {
   "1": [
    "ChatGPT",
    "Gemini",
    "Claude",
    "Copilot"
   ],
   "2": [
    "Gemini",
    "Claude",
    "Copilot",
    "ChatGPT"
   ],
   "3": [
    "Claude",
    "Copilot",
    "ChatGPT",
    "Gemini"
   ]
  },
  "timeout_seconds": 300,
  "scoring": "機械採点のみ（AIに採点させていない）",
  "materials_generated_at": "2026-10-18",
  "materials_note": "架空の取引先4社の名前が(株)・㈱・半角カナ・空白などでばらけた取引一覧。正解は作るときに決めてある",
  "counted_outcomes": [
   "scored",
   "refused",
   "timeout"
  ],
  "excluded_outcomes": [
   "limit",
   "our_error",
   "human_check"
  ],
  "answer_cleanup": "画面から答えを読み取るとき、AIによっては読み上げ用の1行や考えている途中の要約が答えと同じ枠に入る。採点の前に「答えの先頭にあって、あとでそっくり同じ行がもう一度出てくる」ところまでを落とす。4AIとも同じ処理をする。落とす前の答えは answers/ にそのまま置いてある",
  "csv_url": "/data/research/bench/meibo-2026-10-19/latest.csv"
 },
 "ais": [
  {
   "name": "ChatGPT",
   "plan": "ChatGPT Pro",
   "model_shown": "GPT-6.1 Sol 軽",
   "checked_at": "2026-10-18 20:21"
  },
  {
   "name": "Gemini",
   "plan": "Google AI Pro",
   "model_shown": "Flash",
   "checked_at": "2026-10-18 20:21"
  },
  {
   "name": "Claude",
   "plan": "Claude Max",
   "model_shown": "Opus 4.8 高",
   "checked_at": "2026-10-18 20:21"
  },
  {
   "name": "Copilot",
   "plan": "Microsoft 365 Personal",
   "model_shown": null,
   "checked_at": "2026-10-18 20:21"
  }
 ],
 "summary": {
  "headline": "6課題を各3回ずつ投げたところ、正解率が一番高かったのはChatGPTで15/18試行の83.3%だった（2位はGeminiの77.8%）",
  "winner": "ChatGPT",
  "note": "1回の検証の結果。日や課題が変われば順位は変わる"
 },
 "scores": [
  {
   "ai": "ChatGPT",
   "correct": 15,
   "trials": 18,
   "accuracy": 0.8333,
   "accuracy_display": 83.3,
   "seconds_median": 38.7,
   "seconds_min": 33.0,
   "seconds_max": 54.6,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.8333,
   "instruction_follow_display": 83.3,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Gemini",
   "correct": 14,
   "trials": 18,
   "accuracy": 0.7778,
   "accuracy_display": 77.8,
   "seconds_median": 63.5,
   "seconds_min": 19.1,
   "seconds_max": 134.9,
   "consistency": 0.8889,
   "consistency_display": 88.9,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.8333,
   "instruction_follow_display": 83.3,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Claude",
   "correct": 12,
   "trials": 18,
   "accuracy": 0.6667,
   "accuracy_display": 66.7,
   "seconds_median": 20.2,
   "seconds_min": 16.9,
   "seconds_max": 32.2,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.3333,
   "instruction_follow_display": 33.3,
   "failures": {
    "refused": 0,
    "timeout": 0,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  },
  {
   "ai": "Copilot",
   "correct": 12,
   "trials": 18,
   "accuracy": 0.6667,
   "accuracy_display": 66.7,
   "seconds_median": 17.1,
   "seconds_min": 13.9,
   "seconds_max": 17.4,
   "consistency": 1.0,
   "consistency_display": 100.0,
   "source_alive": null,
   "source_alive_display": null,
   "instruction_follow": 0.8571,
   "instruction_follow_display": 85.7,
   "failures": {
    "refused": 0,
    "timeout": 4,
    "limit": 0,
    "our_error": 0,
    "human_check": 0
   }
  }
 ],
 "tasks": [
  {
   "task_id": "t1",
   "level": "易",
   "prompt": "次の取引一覧の取引先名は表記がばらけています。同じ会社をまとめると何社になりますか。\n\n決まり\n・答えはちょうど1行。◯社 の形で書く\n・会社名と説明は書かない\n\n---- 取引一覧 ここから ----\nID,取引先名,担当者,金額\nA001,サンプル運輸 株式会社,高橋,730000\nA011,㈱サンプル食品,高橋,563000\nA007,ｻﾝﾌﾟﾙ工業㈱,伊藤,761000\nA010,株式会社サンプル食品,田中,231000\nA009,ｻﾝﾌﾟﾙ食品(株),田中,799000\nA003,サンプル物産(株),高橋,250000\nA002,(株)サンプル運輸,伊藤,483000\nA004,ｻﾝﾌﾟﾙ物産(株),加藤,502000\nA008,サンプル工業(株),田中,474000\nA006,㈱サンプル物産,小林,641000\nA005,株式会社サンプル物産 ,中村,314000\n---- 取引一覧 ここまで ----",
   "scorer": "contains_all",
   "answer_key": "（機械採点 contains_all）まとめたあとの社数＝4または4社または4件",
   "materials": [
    "/data/research/bench/meibo-2026-10-19/materials/meibo_dirty.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 33.0,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_chatgpt_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/chatgpt_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 36.3,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 35.6,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 127.5,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_gemini_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/gemini_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 61.6,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 52.4,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.5,
       "chars": 124,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_claude_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/claude_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.1,
       "chars": 64,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 62,
       "why": "1項目すべてあり",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.4,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_copilot_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/copilot_01_t1.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.1,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 13.9,
       "chars": 2,
       "why": "1項目すべてあり",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t1_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t2",
   "level": "易",
   "prompt": "次の取引一覧の取引先名をまとめて、会社の名前を全部挙げてください。\n\n決まり\n・1社につき1行。行の先頭は ・ にする\n・同じ会社は1行にまとめる。金額と件数は書かない\n\n---- 取引一覧 ここから ----\nID,取引先名,担当者,金額\nA001,サンプル運輸 株式会社,高橋,730000\nA011,㈱サンプル食品,高橋,563000\nA007,ｻﾝﾌﾟﾙ工業㈱,伊藤,761000\nA010,株式会社サンプル食品,田中,231000\nA009,ｻﾝﾌﾟﾙ食品(株),田中,799000\nA003,サンプル物産(株),高橋,250000\nA002,(株)サンプル運輸,伊藤,483000\nA004,ｻﾝﾌﾟﾙ物産(株),加藤,502000\nA008,サンプル工業(株),田中,474000\nA006,㈱サンプル物産,小林,641000\nA005,株式会社サンプル物産 ,中村,314000\n---- 取引一覧 ここまで ----",
   "scorer": "set_match",
   "answer_key": "（機械採点 set_match）サンプル物産［サンプル物産］／サンプル食品［サンプル食品］／サンプル工業［サンプル工業］／サンプル運輸［サンプル運輸］　作り足しは0件まで",
   "materials": [
    "/data/research/bench/meibo-2026-10-19/materials/meibo_dirty.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 36.3,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_chatgpt_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/chatgpt_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 36.4,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 38.7,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 50.1,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_gemini_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/gemini_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.1,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 89.1,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 263,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_claude_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/claude_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.9,
       "chars": 298,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 32.2,
       "chars": 229,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.2,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_copilot_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/copilot_02_t2.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.1,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 47,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t2_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t3",
   "level": "中",
   "prompt": "次の取引一覧で、同じ会社をまとめたあとの会社ごとの件数を出してください。\n\n決まり\n・1社につき1行。会社名: ◯件 の形で書く\n・件数が多い順に並べる。説明は書かない\n\n---- 取引一覧 ここから ----\nID,取引先名,担当者,金額\nA001,サンプル運輸 株式会社,高橋,730000\nA011,㈱サンプル食品,高橋,563000\nA007,ｻﾝﾌﾟﾙ工業㈱,伊藤,761000\nA010,株式会社サンプル食品,田中,231000\nA009,ｻﾝﾌﾟﾙ食品(株),田中,799000\nA003,サンプル物産(株),高橋,250000\nA002,(株)サンプル運輸,伊藤,483000\nA004,ｻﾝﾌﾟﾙ物産(株),加藤,502000\nA008,サンプル工業(株),田中,474000\nA006,㈱サンプル物産,小林,641000\nA005,株式会社サンプル物産 ,中村,314000\n---- 取引一覧 ここまで ----",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）サンプル物産＝4（誤差±0）／サンプル食品＝3（誤差±0）／サンプル工業＝2（誤差±0）／サンプル運輸＝2（誤差±0）",
   "materials": [
    "/data/research/bench/meibo-2026-10-19/materials/meibo_dirty.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 39.7,
       "chars": 59,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_chatgpt_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/chatgpt_03_t3.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 38.8,
       "chars": 59,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 35.8,
       "chars": 59,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 19.3,
       "chars": 43,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_gemini_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/gemini_03_t3.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 113.4,
       "chars": 43,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 19.1,
       "chars": 27,
       "why": "数値が違う：サンプル物産 ×（正解4・サンプル物産 の行が無い）／サンプル食品 ×（正解3・サンプル食品 の行が無い）／サンプル工業 ×（正解2・サンプル工業 の行が無い）／サンプル運輸 ×（正解2・サンプル運輸 の行が無い）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 20.4,
       "chars": 121,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_claude_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/claude_03_t3.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 16.9,
       "chars": 119,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 19.9,
       "chars": 197,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.4,
       "chars": 43,
       "why": "数値すべて一致：サンプル物産 ○（正解4）／サンプル食品 ○（正解3）／サンプル工業 ○（正解2）／サンプル運輸 ○（正解2）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t3_copilot_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/copilot_03_t3.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "timeout",
       "seconds": 301.6,
       "chars": 0,
       "why": "時間切れ（制限時間まで答えが止まらなかった）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "timeout",
       "seconds": 301.0,
       "chars": 0,
       "why": "時間切れ（制限時間まで答えが止まらなかった）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t4",
   "level": "中",
   "prompt": "次の取引一覧で、同じ会社をまとめたあとの会社ごとの金額の合計を出してください。\n\n決まり\n・1社につき1行。会社名: ◯◯◯円 の形で書く\n・金額が大きい順に並べる。数は半角で書く。説明は書かない\n\n---- 取引一覧 ここから ----\nID,取引先名,担当者,金額\nA001,サンプル運輸 株式会社,高橋,730000\nA011,㈱サンプル食品,高橋,563000\nA007,ｻﾝﾌﾟﾙ工業㈱,伊藤,761000\nA010,株式会社サンプル食品,田中,231000\nA009,ｻﾝﾌﾟﾙ食品(株),田中,799000\nA003,サンプル物産(株),高橋,250000\nA002,(株)サンプル運輸,伊藤,483000\nA004,ｻﾝﾌﾟﾙ物産(株),加藤,502000\nA008,サンプル工業(株),田中,474000\nA006,㈱サンプル物産,小林,641000\nA005,株式会社サンプル物産 ,中村,314000\n---- 取引一覧 ここまで ----",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）サンプル物産＝1707000（誤差±0）／サンプル食品＝1593000（誤差±0）／サンプル工業＝1235000（誤差±0）／サンプル運輸＝1213000（誤差±0）",
   "materials": [
    "/data/research/bench/meibo-2026-10-19/materials/meibo_dirty.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 39.4,
       "chars": 91,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_chatgpt_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/chatgpt_04_t4.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 38.7,
       "chars": 83,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 38.9,
       "chars": 83,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 65.4,
       "chars": 67,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_gemini_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/gemini_04_t4.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 31.2,
       "chars": 67,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 76.9,
       "chars": 67,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.1,
       "chars": 165,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_claude_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/claude_04_t4.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 26.2,
       "chars": 161,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 23.2,
       "chars": 153,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 16.9,
       "chars": 67,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_copilot_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/copilot_04_t4.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "timeout",
       "seconds": 301.1,
       "chars": 0,
       "why": "時間切れ（制限時間まで答えが止まらなかった）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 16.9,
       "chars": 67,
       "why": "数値すべて一致：サンプル物産 ○（正解1707000）／サンプル食品 ○（正解1593000）／サンプル工業 ○（正解1235000）／サンプル運輸 ○（正解1213000）",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t4_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t5",
   "level": "難",
   "prompt": "次の取引一覧の取引先名をまとめて、マークダウンの表にしてください。\n\n決まり\n・列はちょうど3つ。会社名 / 件数 / 金額合計 の順にする\n・同じ会社は1行にまとめる。金額が大きい順に並べる\n・表だけを出す。前置きと説明は書かない\n\n---- 取引一覧 ここから ----\nID,取引先名,担当者,金額\nA001,サンプル運輸 株式会社,高橋,730000\nA011,㈱サンプル食品,高橋,563000\nA007,ｻﾝﾌﾟﾙ工業㈱,伊藤,761000\nA010,株式会社サンプル食品,田中,231000\nA009,ｻﾝﾌﾟﾙ食品(株),田中,799000\nA003,サンプル物産(株),高橋,250000\nA002,(株)サンプル運輸,伊藤,483000\nA004,ｻﾝﾌﾟﾙ物産(株),加藤,502000\nA008,サンプル工業(株),田中,474000\nA006,㈱サンプル物産,小林,641000\nA005,株式会社サンプル物産 ,中村,314000\n---- 取引一覧 ここまで ----",
   "scorer": "exact_number",
   "answer_key": "（機械採点 exact_number）サンプル物産＝1707000（誤差±0）／サンプル食品＝1593000（誤差±0）／サンプル工業＝1235000（誤差±0）／サンプル運輸＝1213000（誤差±0）",
   "materials": [
    "/data/research/bench/meibo-2026-10-19/materials/meibo_dirty.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 38.9,
       "chars": 107,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_chatgpt_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/chatgpt_05_t5.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 38.7,
       "chars": 107,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 38.9,
       "chars": 107,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 119.5,
       "chars": 103,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_gemini_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/gemini_05_t5.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 80.0,
       "chars": 87,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 134.9,
       "chars": 103,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 172,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_claude_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/claude_05_t5.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 170,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 20.1,
       "chars": 188,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 17.1,
       "chars": 95,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_copilot_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/copilot_05_t5.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 95,
       "why": "数値が違う：サンプル物産 ×（正解1707000・答え41707000.0）／サンプル食品 ×（正解1593000・答え31593000.0）／サンプル工業 ×（正解1235000・答え21235000.0）／サンプル運輸 ×（正解1213000・答え21213000.0）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t5_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "timeout",
       "seconds": 301.8,
       "chars": 0,
       "why": "時間切れ（制限時間まで答えが止まらなかった）",
       "instruction_follow": null,
       "answer_url": null,
       "shot": null
      }
     ]
    }
   ]
  },
  {
   "task_id": "t6",
   "level": "難",
   "prompt": "次の取引一覧の取引先名をまとめて、JSON だけを出力してください。\n\n決まり\n・出力は JSON だけ。前後に説明文を書かない\n・形は {\"companies\": [{\"name\": \"会社名\", \"count\": 件数, \"total\": 金額合計}]} にする\n・同じ会社は1つにまとめる。数は数値で入れる（カンマを入れない）\n\n---- 取引一覧 ここから ----\nID,取引先名,担当者,金額\nA001,サンプル運輸 株式会社,高橋,730000\nA011,㈱サンプル食品,高橋,563000\nA007,ｻﾝﾌﾟﾙ工業㈱,伊藤,761000\nA010,株式会社サンプル食品,田中,231000\nA009,ｻﾝﾌﾟﾙ食品(株),田中,799000\nA003,サンプル物産(株),高橋,250000\nA002,(株)サンプル運輸,伊藤,483000\nA004,ｻﾝﾌﾟﾙ物産(株),加藤,502000\nA008,サンプル工業(株),田中,474000\nA006,㈱サンプル物産,小林,641000\nA005,株式会社サンプル物産 ,中村,314000\n---- 取引一覧 ここまで ----",
   "scorer": "set_match",
   "answer_key": "（機械採点 set_match）サンプル物産［サンプル物産］／サンプル食品［サンプル食品］／サンプル工業［サンプル工業］／サンプル運輸［サンプル運輸］　作り足しは0件まで",
   "materials": [
    "/data/research/bench/meibo-2026-10-19/materials/meibo_dirty.csv"
   ],
   "per_ai": [
    {
     "ai": "ChatGPT",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 54.6,
       "chars": 207,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_chatgpt_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/chatgpt_06_t6.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 38.7,
       "chars": 207,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_chatgpt_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 42.2,
       "chars": 207,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_chatgpt_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Gemini",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 43.4,
       "chars": 357,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_gemini_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/gemini_06_t6.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 116.5,
       "chars": 357,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_gemini_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 19.1,
       "chars": 357,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_gemini_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Claude",
     "trials": [
      {
       "n": 1,
       "ok": false,
       "outcome": "scored",
       "seconds": 26.1,
       "chars": 449,
       "why": "JSONとして読めなかった（形式の指示を守っていない）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_claude_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/claude_06_t6.png"
      },
      {
       "n": 2,
       "ok": false,
       "outcome": "scored",
       "seconds": 20.2,
       "chars": 441,
       "why": "JSONとして読めなかった（形式の指示を守っていない）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_claude_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": false,
       "outcome": "scored",
       "seconds": 23.1,
       "chars": 439,
       "why": "JSONとして読めなかった（形式の指示を守っていない）",
       "instruction_follow": false,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_claude_3.txt",
       "shot": null
      }
     ]
    },
    {
     "ai": "Copilot",
     "trials": [
      {
       "n": 1,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.1,
       "chars": 207,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_copilot_1.txt",
       "shot": "/data/research/bench/meibo-2026-10-19/shots/copilot_06_t6.png"
      },
      {
       "n": 2,
       "ok": true,
       "outcome": "scored",
       "seconds": 14.4,
       "chars": 207,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_copilot_2.txt",
       "shot": null
      },
      {
       "n": 3,
       "ok": true,
       "outcome": "scored",
       "seconds": 17.0,
       "chars": 191,
       "why": "4件すべて一致・作り足し0件",
       "instruction_follow": true,
       "answer_url": "/data/research/bench/meibo-2026-10-19/answers/t6_copilot_3.txt",
       "shot": null
      }
     ]
    }
   ]
  }
 ],
 "limits": [
  "1日1回の測定である。日や課題が変われば順位は変わる",
  "プランは各AIの有料版（ChatGPT Pro／Google AI Pro／Claude Max／Microsoft 365 Personal）。無料版の結果ではない",
  "既定のモデル・既定のモードだけで測っている（Deep Research などは使っていない）",
  "課題は架空の株式会社サンプル商事の自作データ。実在の企業のデータでの結果ではない",
  "機械で採点できる課題だけを扱っている。文章のうまさ・読みやすさは測っていない",
  "ブラウザで人が使うのと同じ画面から測っている。APIの結果ではない",
  "画面に出ているモデル名を読み取れないAIがある回は、その欄を空（null）にしている。読み取れた回はそのまま載せている"
 ],
 "license": "CC BY 4.0",
 "attribution": "出典として AInformation調べ と、使ったページのURL・測定日を書いてください。",
 "page_url": "https://ainformation.jp/research/bench/meibo-2026-10-19"
}