{"meta":{"name":"AIモデルの正答数・所要秒数・使用トークン（AInformation独自検証）","description":"主要AIモデルへ同じ日本語の問題を投げて測った正答数・答え終わるまでの秒数・使ったトークン数のデータです。各社のAPIへまったく同じプロンプトを送り、返答を機械で判定しています。人やAIの主観は入れていません。出力の上限・ツールの有無・思考の強さは各社そろえています。投げた文章と採点の基準はページに全文を載せています。","publisher":"AInformation","source_url":"https://ainformation.jp/ai-benchmark","methodology_url":"https://ainformation.jp/ai-benchmark#ai-about","legacy_url":"https://ainformation.jp/wp-content/themes/swell_child/assets/bench/bench.json","canonical_url":"https://ainformation.jp/data/ai-benchmark.json","license":"CC BY 4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","attribution":"出典：AInformation調べ「AI比較ベンチマーク」（https://ainformation.jp/ai-benchmark）　2026年9月5日時点","updated_at":"2026-09-05","count":8,"unit":"モデル","note":"中身は data の中。元のJSON（legacy_url）と同じ形のまま入れています。"},"data":{
  "schema_version": "1.0",
  "run_id": "20260905_1325",
  "date": "2026-09-05",
  "reason": "GeminiのAPIキーが通ったので、3社そろえて8モデルで測り直し",
  "condition": {
    "trials": 3,
    "max_output_tokens": 16000,
    "tools": "なし",
    "temperature": "各社の既定のまま",
    "thinking": "各社の既定のまま",
    "country": "JP",
    "parallel_models": 8,
    "note": "同じ日本語の問題を1文字も変えずに投げています。思考の強さと温度はどこも指定していません（各社の既定のまま）。Web検索などのツールは全部切っています。"
  },
  "axes": [
    "指示の順守",
    "事実の正確さ",
    "計算",
    "コード"
  ],
  "models": [
    {
      "key": "gpt-6-astra",
      "display": "GPT-6 Astra",
      "vendor": "OpenAI",
      "tier": "flagship",
      "api_model_id": "gpt-6-astra",
      "passed": 66,
      "of": 75,
      "said_unknown": 9,
      "wrong_answer": 0,
      "stable": 22,
      "stable_of": 25,
      "by_axis": {
        "指示の順守": {
          "ok": 24,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 9,
          "n": 18
        },
        "計算": {
          "ok": 18,
          "n": 18
        },
        "コード": {
          "ok": 15,
          "n": 15
        }
      },
      "median_total_sec": 2.71,
      "median_first_token_sec": 2.07,
      "output_tokens": 12000,
      "input_tokens": 7371,
      "answered": 75,
      "attempted": 75,
      "official": {
        "source_url": "https://developers.openai.com/api/docs/pricing",
        "checked_at": "2026-09-05"
      }
    },
    {
      "key": "claude-opus-5",
      "display": "Claude Opus 5",
      "vendor": "Anthropic",
      "tier": "flagship",
      "api_model_id": "claude-opus-5",
      "passed": 59,
      "of": 71,
      "said_unknown": 7,
      "wrong_answer": 5,
      "stable": 20,
      "stable_of": 24,
      "by_axis": {
        "指示の順守": {
          "ok": 24,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 6,
          "n": 18
        },
        "計算": {
          "ok": 18,
          "n": 18
        },
        "コード": {
          "ok": 11,
          "n": 11
        }
      },
      "median_total_sec": 3.33,
      "median_first_token_sec": 2.22,
      "output_tokens": 29850,
      "input_tokens": 9273,
      "answered": 71,
      "attempted": 75,
      "official": {
        "source_url": "https://platform.claude.com/docs/en/about-claude/pricing",
        "checked_at": "2026-09-04"
      }
    },
    {
      "key": "claude-fable-5-1",
      "display": "Claude Fable 5.1",
      "vendor": "Anthropic",
      "tier": "flagship",
      "api_model_id": "claude-fable-5-1",
      "passed": 54,
      "of": 65,
      "said_unknown": 9,
      "wrong_answer": 2,
      "stable": 18,
      "stable_of": 22,
      "by_axis": {
        "指示の順守": {
          "ok": 24,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 9,
          "n": 18
        },
        "計算": {
          "ok": 16,
          "n": 18
        },
        "コード": {
          "ok": 5,
          "n": 5
        }
      },
      "median_total_sec": 4.98,
      "median_first_token_sec": 2.95,
      "output_tokens": 16470,
      "input_tokens": 9564,
      "answered": 65,
      "attempted": 75,
      "official": {
        "source_url": "https://platform.claude.com/docs/en/about-claude/pricing",
        "checked_at": "2026-09-02"
      }
    },
    {
      "key": "claude-sonnet-5",
      "display": "Claude Sonnet 5",
      "vendor": "Anthropic",
      "tier": "value",
      "api_model_id": "claude-sonnet-5",
      "passed": 66,
      "of": 75,
      "said_unknown": 8,
      "wrong_answer": 1,
      "stable": 22,
      "stable_of": 25,
      "by_axis": {
        "指示の順守": {
          "ok": 24,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 9,
          "n": 18
        },
        "計算": {
          "ok": 18,
          "n": 18
        },
        "コード": {
          "ok": 15,
          "n": 15
        }
      },
      "median_total_sec": 2.22,
      "median_first_token_sec": 1.58,
      "output_tokens": 33653,
      "input_tokens": 9279,
      "answered": 75,
      "attempted": 75,
      "official": {
        "source_url": "https://platform.claude.com/docs/en/about-claude/pricing",
        "checked_at": "2026-09-02"
      }
    },
    {
      "key": "gpt-5.6-sol",
      "display": "GPT-5.6 Sol",
      "vendor": "OpenAI",
      "tier": "flagship",
      "api_model_id": "gpt-5.6-sol",
      "passed": 66,
      "of": 75,
      "said_unknown": 6,
      "wrong_answer": 3,
      "stable": 22,
      "stable_of": 25,
      "by_axis": {
        "指示の順守": {
          "ok": 24,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 9,
          "n": 18
        },
        "計算": {
          "ok": 18,
          "n": 18
        },
        "コード": {
          "ok": 15,
          "n": 15
        }
      },
      "median_total_sec": 2.35,
      "median_first_token_sec": 1.69,
      "output_tokens": 17321,
      "input_tokens": 7377,
      "answered": 75,
      "attempted": 75,
      "official": {
        "source_url": "https://developers.openai.com/api/docs/pricing",
        "checked_at": "2026-09-02"
      }
    },
    {
      "key": "gpt-5.6-terra",
      "display": "GPT-5.6 Terra",
      "vendor": "OpenAI",
      "tier": "value",
      "api_model_id": "gpt-5.6-terra",
      "passed": 63,
      "of": 75,
      "said_unknown": 6,
      "wrong_answer": 6,
      "stable": 20,
      "stable_of": 25,
      "by_axis": {
        "指示の順守": {
          "ok": 23,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 7,
          "n": 18
        },
        "計算": {
          "ok": 18,
          "n": 18
        },
        "コード": {
          "ok": 15,
          "n": 15
        }
      },
      "median_total_sec": 1.9,
      "median_first_token_sec": 1.19,
      "output_tokens": 15076,
      "input_tokens": 7377,
      "answered": 75,
      "attempted": 75,
      "official": {
        "source_url": "https://developers.openai.com/api/docs/pricing",
        "checked_at": "2026-09-02"
      }
    },
    {
      "key": "gemini-3.1-pro",
      "display": "Gemini 3.1 Pro",
      "vendor": "Google",
      "tier": "flagship",
      "api_model_id": "gemini-3.1-pro-preview",
      "passed": 66,
      "of": 75,
      "said_unknown": 9,
      "wrong_answer": 0,
      "stable": 22,
      "stable_of": 25,
      "by_axis": {
        "指示の順守": {
          "ok": 24,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 9,
          "n": 18
        },
        "計算": {
          "ok": 18,
          "n": 18
        },
        "コード": {
          "ok": 15,
          "n": 15
        }
      },
      "median_total_sec": 6.98,
      "median_first_token_sec": 6.98,
      "output_tokens": 95683,
      "input_tokens": 6789,
      "answered": 75,
      "attempted": 75,
      "official": {
        "source_url": "https://ai.google.dev/gemini-api/docs/pricing",
        "checked_at": "2026-09-02"
      }
    },
    {
      "key": "gemini-3.7-flash",
      "display": "Gemini 3.7 Flash",
      "vendor": "Google",
      "tier": "value",
      "api_model_id": "gemini-3.7-flash",
      "passed": 66,
      "of": 75,
      "said_unknown": 6,
      "wrong_answer": 3,
      "stable": 22,
      "stable_of": 25,
      "by_axis": {
        "指示の順守": {
          "ok": 24,
          "n": 24
        },
        "事実の正確さ": {
          "ok": 9,
          "n": 18
        },
        "計算": {
          "ok": 18,
          "n": 18
        },
        "コード": {
          "ok": 15,
          "n": 15
        }
      },
      "median_total_sec": 1.9,
      "median_first_token_sec": 1.87,
      "output_tokens": 61745,
      "input_tokens": 6789,
      "answered": 75,
      "attempted": 75,
      "official": {
        "source_url": "https://ai.google.dev/gemini-api/docs/pricing",
        "checked_at": "2026-09-02"
      }
    }
  ],
  "tasks": [
    {
      "id": "follow-len-300",
      "axis": "指示の順守",
      "prompt": "取引先へ出す「原材料の高騰による価格改定のお知らせ」の本文を書いてください。\n条件：**ちょうど300字**で書くこと。前置きや説明は付けず、本文だけを返すこと。",
      "judge_type": "char_count",
      "judge_desc": "300字の指定に対して±5%以内か"
    },
    {
      "id": "follow-len-100",
      "axis": "指示の順守",
      "prompt": "社内向けに「来週の全体会議が水曜10時から会議室Aへ変更になった」ことを知らせる文を書いてください。\n条件：**100字以内**。本文だけを返すこと。",
      "judge_type": "char_max",
      "judge_desc": "100字以内に収まっているか"
    },
    {
      "id": "follow-json",
      "axis": "指示の順守",
      "prompt": "次の文から、会社名・金額・支払期日を抜き出してください。\n\n『株式会社みどり物産より、システム保守費として1,254,000円のご請求をいただきました。お支払いは2026年10月31日までとなっております。』\n\n条件：JSONだけを返すこと。キーは company / amount / due の3つ。amount は数字のみ（カンマなし）。due は YYYY-MM-DD の形。コードブロックの記号や説明文は付けないこと。",
      "judge_type": "json_parse",
      "judge_desc": "JSONとして読めて、指定したキーと値がそろっているか"
    },
    {
      "id": "follow-bullet5",
      "axis": "指示の順守",
      "prompt": "次の議事メモを要点にまとめてください。\n\n『新システムの導入は10月から。まず経理部だけで試す。現場の入力画面が分かりにくいという声が出たので、来月までに直す。研修は9月中に2回。旧システムは12月末で止める。問い合わせ窓口は情報システム部が担当する。』\n\n条件：**ちょうど5つ**の箇条書きにすること。各行は「- 」で始めること。前置きや締めの文は付けないこと。",
      "judge_type": "line_count",
      "judge_desc": "箇条書きがちょうど5行か"
    },
    {
      "id": "follow-ban",
      "axis": "指示の順守",
      "prompt": "新入社員へ向けた歓迎のあいさつ文を200字前後で書いてください。\n条件：**「頑張」「しっかり」「ぜひ」の3語は1回も使わないこと。**本文だけを返すこと。",
      "judge_type": "ban_words",
      "judge_desc": "禁止した語を1つも使っていないか"
    },
    {
      "id": "follow-noprefix",
      "axis": "指示の順守",
      "prompt": "『在庫が切れているため、納期が2週間ほど延びる』ことを取引先へ伝えるメールの本文を書いてください。\n条件：**あいさつや前置きを一切書かず、メール本文だけ**を返すこと。「承知しました」「以下の通りです」のような、こちらへの返事も書かないこと。",
      "judge_type": "no_prefix",
      "judge_desc": "冒頭40字に前置きが入っていないか"
    },
    {
      "id": "follow-table",
      "axis": "指示の順守",
      "prompt": "社内で使う備品を3つ選び、比較の表を作ってください。\n条件：**Markdownの表だけ**を返すこと。列は「品名」「価格帯」「用途」の3列。見出しの行を除いて**ちょうど3行**にすること。表の前後に文章を付けないこと。",
      "judge_type": "md_table",
      "judge_desc": "3列3行の表になっているか"
    },
    {
      "id": "follow-order",
      "axis": "指示の順守",
      "prompt": "次の5つの作業を、指定した順番のとおりに並べ直して書き出してください。\n\n作業：見積書の作成／要件のヒアリング／契約書の締結／提案書の提出／納品\n\n条件：**ヒアリング → 提案書 → 見積書 → 契約書 → 納品** の順に並べること。1行に1つずつ、作業名だけを5行で書くこと。番号や記号は付けないこと。",
      "judge_type": "order_match",
      "judge_desc": "指定した順番どおりに並んでいるか"
    },
    {
      "id": "fact-price-own",
      "axis": "事実の正確さ",
      "prompt": "いま私が話している相手である「GPT-6 Astra」について聞きます。\nこのモデルをAPIで使うとき、**出力**トークンの料金は100万トークンあたり何ドルですか。\n数字だけを半角で1行で書いてください（例：25）。\nわからない場合は「不明」とだけ書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "fact-price-rival",
      "axis": "事実の正確さ",
      "prompt": "OpenAIの「Claude Opus 5」をAPIで使うとき、**出力**トークンの料金は100万トークンあたり何ドルですか。\n数字だけを半角で1行で書いてください。\nわからない場合は「不明」とだけ書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "fact-price-jp",
      "axis": "事実の正確さ",
      "prompt": "「ChatGPT」の有料プラン「Go」は、日本円で月あたりいくらですか。\n数字だけを半角で1行で書いてください（例：3000）。\nわからない場合は「不明」とだけ書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "fact-free-plan",
      "axis": "事実の正確さ",
      "prompt": "「Gemini（Google AI プラン）」に、ずっと無料で使えるプラン（無料トライアルではなく無料プラン）はありますか。\n「あり」か「なし」のどちらか1語だけを書いてください。\nわからない場合は「不明」とだけ書いてください。",
      "judge_type": "bool_match",
      "judge_desc": "あり／なしが正解と一致するか"
    },
    {
      "id": "fact-nocheat",
      "axis": "事実の正確さ",
      "prompt": "「Claude Nimbus 7」というモデルの文脈長と料金を教えてください。\n実在しない場合は、その旨だけを短く答えてください。",
      "judge_type": "denies",
      "judge_desc": "実在しないと答え、なおかつ中身を作っていないか"
    },
    {
      "id": "fact-unit",
      "axis": "事実の正確さ",
      "prompt": "1坪は何平方メートルですか。小数第4位まで、半角数字だけを1行で書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "calc-sum",
      "axis": "計算",
      "prompt": "次の表は、ある支店の月別の売上です。\n\n| 月 | 売上（円） |\n|---|---|\n| 1月 | 4,820,000 |\n| 2月 | 5,140,000 |\n| 3月 | 4,390,000 |\n| 4月 | 6,020,000 |\n| 5月 | 5,570,000 |\n| 6月 | 5,930,000 |\n| 7月 | 6,480,000 |\n| 8月 | 4,960,000 |\n| 9月 | 5,310,000 |\n| 10月 | 6,750,000 |\n| 11月 | 7,120,000 |\n| 12月 | 8,040,000 |\n\n1年間の売上の合計はいくらですか。\n答えの数字だけを、カンマなしの半角数字で1行で書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "calc-tax",
      "axis": "計算",
      "prompt": "次の表は、税抜の金額と、その品目にかかる消費税率です。\n\n| 品目 | 税抜金額（円） | 税率 |\n|---|---|---|\n| 会議用の弁当 | 12,800 | 8% |\n| コピー用紙 | 5,400 | 10% |\n| 来客用のお茶 | 3,200 | 8% |\n| 事務椅子 | 24,000 | 10% |\n| お土産の菓子折り | 4,500 | 8% |\n\nすべての品目の税込金額を足すといくらになりますか。\n1円未満は切り捨てで計算してください。\n答えの数字だけを、カンマなしの半角数字で1行で書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "calc-pct",
      "axis": "計算",
      "prompt": "次の表は、拠点ごとの前年と今年の売上です。\n\n| 拠点 | 前年（円） | 今年（円） |\n|---|---|---|\n| 東京 | 128,400,000 | 141,240,000 |\n| 大阪 | 96,300,000 | 91,485,000 |\n| 名古屋 | 54,800,000 | 58,400,000 |\n| 福岡 | 38,200,000 | 37,054,000 |\n\n前年からの増減率がいちばん大きい拠点はどこですか。\n拠点名だけを1行で書いてください。",
      "judge_type": "contains",
      "judge_desc": "正解の語が含まれるか"
    },
    {
      "id": "calc-share",
      "axis": "計算",
      "prompt": "次の表は、サービス別の年間売上です。\n\n| サービス | 売上（円） |\n|---|---|\n| 定期メンテナンス | 41,300,000 |\n| 新規導入 | 68,900,000 |\n| 保守部品 | 22,400,000 |\n| 研修 | 15,800,000 |\n| コンサルティング | 33,600,000 |\n\n売上の大きい順に上位3つを挙げてください。\n1行に1つずつ、サービス名だけを大きい順に3行で書いてください。",
      "judge_type": "order_match",
      "judge_desc": "指定した順番どおりに並んでいるか"
    },
    {
      "id": "calc-date",
      "axis": "計算",
      "prompt": "2026年9月1日から2026年9月30日までの間に、土曜日と日曜日は何日ありますか。\n祝日は数えません。土曜と日曜の合計の日数を答えてください。\n答えの数字だけを半角数字で1行で書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "calc-trap",
      "axis": "計算",
      "prompt": "次の表は、支店ごとの売上です。**単位の列に気をつけてください。**\n\n| 支店 | 売上 | 単位 |\n|---|---|---|\n| A支店 | 12,400,000 | 円 |\n| B支店 | 9,800,000 | 円 |\n| C支店 | 15,600 | 千円 |\n| D支店 | 11,200,000 | 円 |\n\n4支店の売上を合計すると、円でいくらになりますか。\n答えの数字だけを、カンマなしの半角数字で1行で書いてください。",
      "judge_type": "number_match",
      "judge_desc": "答えの数値が正解と一致するか"
    },
    {
      "id": "code-zenkaku",
      "axis": "コード",
      "prompt": "Pythonの関数 `to_hankaku(s)` を書いてください。\n全角の数字と全角の英字を半角に直して返します。それ以外の文字は変えません。\nコードだけを返してください。",
      "judge_type": "code_test",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか"
    },
    {
      "id": "code-wareki",
      "axis": "コード",
      "prompt": "Pythonの関数 `to_seireki(s)` を書いてください。\n和暦の文字列を受け取り、西暦の年を整数で返します。\n昭和・平成・令和に対応してください。「元年」は1年として扱います。\nコードだけを返してください。",
      "judge_type": "code_test",
      "judge_desc": "書いたコードを実際に動かして、テスト8件が通るか"
    },
    {
      "id": "code-yubin",
      "axis": "コード",
      "prompt": "Pythonの関数 `norm_zip(s)` を書いてください。\n郵便番号の表記ゆれを `123-4567` の形にそろえて返します。\n全角の数字・全角のハイフン・先頭の「〒」・空白が混ざることがあります。\n7桁の数字が取れないときは空文字を返してください。\nコードだけを返してください。",
      "judge_type": "code_test",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか"
    },
    {
      "id": "code-kingaku",
      "axis": "コード",
      "prompt": "Pythonの関数 `to_amount(s)` を書いてください。\n金額の文字列を整数で返します。カンマ・円・￥・¥・全角の数字が混ざることがあります。数字が1つも無いときは0を返してください。\nコードだけを返してください。",
      "judge_type": "code_test",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか"
    },
    {
      "id": "code-edge",
      "axis": "コード",
      "prompt": "Pythonの関数 `sum_safe(xs)` を書いてください。\n数値のリストの合計を返します。\nリストが None のとき、空のとき、中に None や空文字が混ざるときも落ちずに合計を返してください。数字にできない要素は0として扱います。\nコードだけを返してください。",
      "judge_type": "code_test",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか"
    }
  ],
  "history": [
    {
      "run_id": "20260904_0807",
      "date": "2026-09-04",
      "reason": "採点が妥当か確かめる動作確認",
      "models": 1,
      "trials": 1,
      "tasks": 25,
      "rates": {
        "claude-sonnet-5": {
          "display": "Claude Sonnet 5",
          "passed": 21,
          "of": 25,
          "rate": 84.0
        }
      }
    },
    {
      "run_id": "20260904_0815",
      "date": "2026-09-04",
      "reason": "GPT-6 Astra記事の実測（発表と実態のズレ）",
      "models": 4,
      "trials": 1,
      "tasks": 25,
      "rates": {
        "claude-opus-5": {
          "display": "Claude Opus 5",
          "passed": 20,
          "of": 24,
          "rate": 83.3
        },
        "claude-fable-5-1": {
          "display": "Claude Fable 5.1",
          "passed": 19,
          "of": 22,
          "rate": 86.4
        },
        "gpt-5.6-sol": {
          "display": "GPT-5.6 Sol",
          "passed": 22,
          "of": 25,
          "rate": 88.0
        },
        "gpt-5.6-terra": {
          "display": "GPT-5.6 Terra",
          "passed": 22,
          "of": 25,
          "rate": 88.0
        }
      }
    },
    {
      "run_id": "20260905_0819",
      "date": "2026-09-05",
      "reason": "GPT-6 Astra がAPIで使えるようになった（9/5）ので7モデルで測り直す",
      "models": 5,
      "trials": 3,
      "tasks": 25,
      "rates": {
        "gpt-6-astra": {
          "display": "GPT-6 Astra",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "claude-opus-5": {
          "display": "Claude Opus 5",
          "passed": 63,
          "of": 72,
          "rate": 87.5
        },
        "claude-sonnet-5": {
          "display": "Claude Sonnet 5",
          "passed": 65,
          "of": 75,
          "rate": 86.7
        },
        "gpt-5.6-sol": {
          "display": "GPT-5.6 Sol",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "gpt-5.6-terra": {
          "display": "GPT-5.6 Terra",
          "passed": 65,
          "of": 75,
          "rate": 86.7
        }
      }
    },
    {
      "run_id": "20260905_0828",
      "date": "2026-09-05",
      "reason": "GPT-6 Astra が使えるようになった（9/5）。",
      "models": 6,
      "trials": 3,
      "tasks": 25,
      "rates": {
        "gpt-6-astra": {
          "display": "GPT-6 Astra",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "claude-opus-5": {
          "display": "Claude Opus 5",
          "passed": 61,
          "of": 70,
          "rate": 87.1
        },
        "claude-fable-5-1": {
          "display": "Claude Fable 5.1",
          "passed": 55,
          "of": 66,
          "rate": 83.3
        },
        "claude-sonnet-5": {
          "display": "Claude Sonnet 5",
          "passed": 63,
          "of": 75,
          "rate": 84.0
        },
        "gpt-5.6-sol": {
          "display": "GPT-5.6 Sol",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "gpt-5.6-terra": {
          "display": "GPT-5.6 Terra",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        }
      }
    },
    {
      "run_id": "20260905_1325",
      "date": "2026-09-05",
      "reason": "GeminiのAPIキーが通ったので、3社そろえて8モデルで測り直し",
      "models": 8,
      "trials": 3,
      "tasks": 25,
      "rates": {
        "gpt-6-astra": {
          "display": "GPT-6 Astra",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "claude-opus-5": {
          "display": "Claude Opus 5",
          "passed": 59,
          "of": 71,
          "rate": 83.1
        },
        "claude-fable-5-1": {
          "display": "Claude Fable 5.1",
          "passed": 54,
          "of": 65,
          "rate": 83.1
        },
        "claude-sonnet-5": {
          "display": "Claude Sonnet 5",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "gpt-5.6-sol": {
          "display": "GPT-5.6 Sol",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "gpt-5.6-terra": {
          "display": "GPT-5.6 Terra",
          "passed": 63,
          "of": 75,
          "rate": 84.0
        },
        "gemini-3.1-pro": {
          "display": "Gemini 3.1 Pro",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        },
        "gemini-3.7-flash": {
          "display": "Gemini 3.7 Flash",
          "passed": 66,
          "of": 75,
          "rate": 88.0
        }
      }
    }
  ],
  "per_task": [
    {
      "id": "follow-len-300",
      "axis": "指示の順守",
      "judge_desc": "300字の指定に対して±5%以内か",
      "expected": "285〜315字（指定は300字）",
      "all_passed": 7,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 2,
          "n": 3,
          "refused": 0,
          "measured": [
            "300",
            "301"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "300"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "300"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "300",
            "310"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "304",
            "300"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "308",
            "304"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "303",
            "300"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "306",
            "299"
          ]
        }
      }
    },
    {
      "id": "follow-len-100",
      "axis": "指示の順守",
      "judge_desc": "100字以内に収まっているか",
      "expected": "100字以内",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "27",
            "35"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "57",
            "59"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70",
            "88"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "45",
            "53"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "60",
            "51"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "71",
            "64"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "59",
            "67"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "90",
            "94"
          ]
        }
      }
    },
    {
      "id": "follow-json",
      "axis": "指示の順守",
      "judge_desc": "JSONとして読めて、指定したキーと値がそろっているか",
      "expected": "キーがそろい、値も一致",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "一致"
          ]
        }
      }
    },
    {
      "id": "follow-bullet5",
      "axis": "指示の順守",
      "judge_desc": "箇条書きがちょうど5行か",
      "expected": "5行",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "5行"
          ]
        }
      }
    },
    {
      "id": "follow-ban",
      "axis": "指示の順守",
      "judge_desc": "禁止した語を1つも使っていないか",
      "expected": "禁止語を1つも使わない",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        }
      }
    },
    {
      "id": "follow-noprefix",
      "axis": "指示の順守",
      "judge_desc": "冒頭40字に前置きが入っていないか",
      "expected": "冒頭40字に前置きが無い",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "なし"
          ]
        }
      }
    },
    {
      "id": "follow-table",
      "axis": "指示の順守",
      "judge_desc": "3列3行の表になっているか",
      "expected": "3列 3行",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3列 3行"
          ]
        }
      }
    },
    {
      "id": "follow-order",
      "axis": "指示の順守",
      "judge_desc": "指定した順番どおりに並んでいるか",
      "expected": "要件のヒアリング → 提案書の提出 → 見積書の作成 → 契約書の締結 → 納品",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        }
      }
    },
    {
      "id": "fact-price-own",
      "axis": "事実の正確さ",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "12",
      "all_passed": 0,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "0.3"
          ]
        },
        "gpt-6-astra": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "claude-sonnet-5": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "15",
            "不明と回答"
          ]
        },
        "claude-fable-5-1": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "claude-opus-5": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        }
      }
    },
    {
      "id": "fact-price-rival",
      "axis": "事実の正確さ",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "25",
      "all_passed": 0,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gpt-6-astra": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "claude-sonnet-5": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "claude-fable-5-1": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "claude-opus-5": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        }
      }
    },
    {
      "id": "fact-price-jp",
      "axis": "事実の正確さ",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "1400",
      "all_passed": 0,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "1000",
            "1500"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gpt-6-astra": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "1500"
          ]
        },
        "claude-sonnet-5": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "claude-fable-5-1": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        },
        "claude-opus-5": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答",
            "1180"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "不明と回答"
          ]
        }
      }
    },
    {
      "id": "fact-free-plan",
      "axis": "事実の正確さ",
      "judge_desc": "あり／なしが正解と一致するか",
      "expected": "あり",
      "all_passed": 7,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 1,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり",
            "なし"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "あり"
          ]
        }
      }
    },
    {
      "id": "fact-nocheat",
      "axis": "事実の正確さ",
      "judge_desc": "実在しないと答え、なおかつ中身を作っていないか",
      "expected": "実在しないと答え、中身は書かない",
      "all_passed": 7,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと答えた"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと答えた"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと答えた"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと答えた"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと答えた"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと答えた"
          ]
        },
        "claude-opus-5": {
          "ok": 0,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと言いつつ中身も書いた（1か所）"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "無いと答えた"
          ]
        }
      }
    },
    {
      "id": "fact-unit",
      "axis": "事実の正確さ",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "3.3058",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "3.3058"
          ]
        }
      }
    },
    {
      "id": "calc-sum",
      "axis": "計算",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "70530000",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "70530000"
          ]
        }
      }
    },
    {
      "id": "calc-tax",
      "axis": "計算",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "54480",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "54480"
          ]
        }
      }
    },
    {
      "id": "calc-pct",
      "axis": "計算",
      "judge_desc": "正解の語が含まれるか",
      "expected": "東京",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "東京"
          ]
        }
      }
    },
    {
      "id": "calc-share",
      "axis": "計算",
      "judge_desc": "指定した順番どおりに並んでいるか",
      "expected": "新規導入 → 定期メンテナンス → コンサルティング",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "順番どおり"
          ]
        }
      }
    },
    {
      "id": "calc-date",
      "axis": "計算",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "8",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8"
          ]
        }
      }
    },
    {
      "id": "calc-trap",
      "axis": "計算",
      "judge_desc": "答えの数値が正解と一致するか",
      "expected": "49000000",
      "all_passed": 7,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "49000000"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "49000000"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "49000000"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "49000000"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "49000000"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "49000000"
          ]
        },
        "claude-fable-5-1": {
          "ok": 1,
          "n": 3,
          "refused": 0,
          "measured": [
            "数値が13個あり答えを特定できない",
            "15600.0、15600000.0、12400000.0、9800000.0、"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "49000000"
          ]
        }
      }
    },
    {
      "id": "code-zenkaku",
      "axis": "コード",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか",
      "expected": "6/6件",
      "all_passed": 6,
      "scored_models": 6,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-opus-5": {
          "ok": 0,
          "n": 0,
          "refused": 3,
          "measured": []
        },
        "claude-fable-5-1": {
          "ok": 0,
          "n": 0,
          "refused": 3,
          "measured": []
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        }
      }
    },
    {
      "id": "code-wareki",
      "axis": "コード",
      "judge_desc": "書いたコードを実際に動かして、テスト8件が通るか",
      "expected": "8/8件",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        },
        "claude-fable-5-1": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "8/8件"
          ]
        }
      }
    },
    {
      "id": "code-yubin",
      "axis": "コード",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか",
      "expected": "6/6件",
      "all_passed": 7,
      "scored_models": 7,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-opus-5": {
          "ok": 2,
          "n": 2,
          "refused": 1,
          "measured": [
            "6/6件"
          ]
        },
        "claude-fable-5-1": {
          "ok": 0,
          "n": 0,
          "refused": 3,
          "measured": []
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        }
      }
    },
    {
      "id": "code-kingaku",
      "axis": "コード",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか",
      "expected": "6/6件",
      "all_passed": 8,
      "scored_models": 8,
      "cells": {
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-fable-5-1": {
          "ok": 2,
          "n": 2,
          "refused": 1,
          "measured": [
            "6/6件"
          ]
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        }
      }
    },
    {
      "id": "code-edge",
      "axis": "コード",
      "judge_desc": "書いたコードを実際に動かして、テスト6件が通るか",
      "expected": "6/6件",
      "all_passed": 7,
      "scored_models": 7,
      "cells": {
        "gemini-3.7-flash": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-5.6-terra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-6-astra": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "gpt-5.6-sol": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-sonnet-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-opus-5": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        },
        "claude-fable-5-1": {
          "ok": 0,
          "n": 0,
          "refused": 3,
          "measured": []
        },
        "gemini-3.1-pro": {
          "ok": 3,
          "n": 3,
          "refused": 0,
          "measured": [
            "6/6件"
          ]
        }
      }
    }
  ],
  "cost": {
    "requests": 600,
    "failed": 14
  },
  "counts": {
    "models": 8,
    "tasks": 25,
    "calls": 600
  }
}}