{"meta":{"name":"Correct answers, seconds taken and tokens used by AI models (original tests by AInformation)","description":"Data on the number of correct answers, the seconds until each answer was finished and the tokens used, measured by giving major AI models the same questions in Japanese. We send exactly the same prompts to each company's API and judge the replies automatically. No subjective judgment by people or AI is involved. Output limits, tool use and reasoning effort are set the same for every company. The full text of the prompts and the scoring criteria are on the page.","language":"en","publisher":"AInformation","source_url":"https://ainformation.jp/en/ai-model-benchmark","methodology_url":"https://ainformation.jp/en/ai-model-benchmark#ai-about","original_url":"https://ainformation.jp/data/ai-benchmark.json","legacy_url":"https://ainformation.jp/wp-content/themes/swell_child/assets/bench/bench.json","canonical_url":"https://ainformation.jp/en/data/ai-benchmark.json","license":"CC BY 4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","attribution":"Source: AInformation, “AI Model Benchmark” (https://ainformation.jp/en/ai-model-benchmark), as of September 5, 2026","updated_at":"2026-09-05","count":8,"unit":" models","note":"The contents are in data, in the same format as the original Japanese JSON (original_url), with only the text translated. Numbers, prices, dates and URLs are the same as in the Japanese version."},"data":{"schema_version":"1.0","run_id":"20260905_1325","date":"2026-09-05","reason":"Gemini's API key started working, so we re-measured 8 models from all three companies","condition":{"trials":3,"max_output_tokens":16000,"tools":"None","temperature":"Each company's default setting","thinking":"Each company's default setting","country":"JP","parallel_models":8,"note":"We submit the same Japanese questions without changing a single character. We don't set reasoning strength or temperature for any provider (each company's default is used). All tools such as web search are turned off."},"axes":["Instruction compliance","Factual accuracy","Calculation","Code"],"models":[{"key":"gpt-6-astra","display":"GPT-6 Astra","vendor":"OpenAI","tier":"flagship","api_model_id":"gpt-6-astra","passed":66,"of":75,"said_unknown":9,"wrong_answer":0,"stable":22,"stable_of":25,"by_axis":{"指示の順守":{"ok":24,"n":24},"事実の正確さ":{"ok":9,"n":18},"計算":{"ok":18,"n":18},"コード":{"ok":15,"n":15}},"median_total_sec":2.70999999999999996447286321199499070644378662109375,"median_first_token_sec":2.069999999999999840127884453977458178997039794921875,"output_tokens":12000,"input_tokens":7371,"answered":75,"attempted":75,"official":{"source_url":"https://developers.openai.com/api/docs/pricing","checked_at":"2026-09-05"}},{"key":"claude-opus-5","display":"Claude Opus 5","vendor":"Anthropic","tier":"flagship","api_model_id":"claude-opus-5","passed":59,"of":71,"said_unknown":7,"wrong_answer":5,"stable":20,"stable_of":24,"by_axis":{"指示の順守":{"ok":24,"n":24},"事実の正確さ":{"ok":6,"n":18},"計算":{"ok":18,"n":18},"コード":{"ok":11,"n":11}},"median_total_sec":3.3300000000000000710542735760100185871124267578125,"median_first_token_sec":2.220000000000000195399252334027551114559173583984375,"output_tokens":29850,"input_tokens":9273,"answered":71,"attempted":75,"official":{"source_url":"https://platform.claude.com/docs/en/about-claude/pricing","checked_at":"2026-09-04"}},{"key":"claude-fable-5-1","display":"Claude Fable 5.1","vendor":"Anthropic","tier":"flagship","api_model_id":"claude-fable-5-1","passed":54,"of":65,"said_unknown":9,"wrong_answer":2,"stable":18,"stable_of":22,"by_axis":{"指示の順守":{"ok":24,"n":24},"事実の正確さ":{"ok":9,"n":18},"計算":{"ok":16,"n":18},"コード":{"ok":5,"n":5}},"median_total_sec":4.980000000000000426325641456060111522674560546875,"median_first_token_sec":2.95000000000000017763568394002504646778106689453125,"output_tokens":16470,"input_tokens":9564,"answered":65,"attempted":75,"official":{"source_url":"https://platform.claude.com/docs/en/about-claude/pricing","checked_at":"2026-09-02"}},{"key":"claude-sonnet-5","display":"Claude Sonnet 5","vendor":"Anthropic","tier":"value","api_model_id":"claude-sonnet-5","passed":66,"of":75,"said_unknown":8,"wrong_answer":1,"stable":22,"stable_of":25,"by_axis":{"指示の順守":{"ok":24,"n":24},"事実の正確さ":{"ok":9,"n":18},"計算":{"ok":18,"n":18},"コード":{"ok":15,"n":15}},"median_total_sec":2.220000000000000195399252334027551114559173583984375,"median_first_token_sec":1.5800000000000000710542735760100185871124267578125,"output_tokens":33653,"input_tokens":9279,"answered":75,"attempted":75,"official":{"source_url":"https://platform.claude.com/docs/en/about-claude/pricing","checked_at":"2026-09-02"}},{"key":"gpt-5.6-sol","display":"GPT-5.6 Sol","vendor":"OpenAI","tier":"flagship","api_model_id":"gpt-5.6-sol","passed":66,"of":75,"said_unknown":6,"wrong_answer":3,"stable":22,"stable_of":25,"by_axis":{"指示の順守":{"ok":24,"n":24},"事実の正確さ":{"ok":9,"n":18},"計算":{"ok":18,"n":18},"コード":{"ok":15,"n":15}},"median_total_sec":2.350000000000000088817841970012523233890533447265625,"median_first_token_sec":1.689999999999999946709294817992486059665679931640625,"output_tokens":17321,"input_tokens":7377,"answered":75,"attempted":75,"official":{"source_url":"https://developers.openai.com/api/docs/pricing","checked_at":"2026-09-02"}},{"key":"gpt-5.6-terra","display":"GPT-5.6 Terra","vendor":"OpenAI","tier":"value","api_model_id":"gpt-5.6-terra","passed":63,"of":75,"said_unknown":6,"wrong_answer":6,"stable":20,"stable_of":25,"by_axis":{"指示の順守":{"ok":23,"n":24},"事実の正確さ":{"ok":7,"n":18},"計算":{"ok":18,"n":18},"コード":{"ok":15,"n":15}},"median_total_sec":1.899999999999999911182158029987476766109466552734375,"median_first_token_sec":1.189999999999999946709294817992486059665679931640625,"output_tokens":15076,"input_tokens":7377,"answered":75,"attempted":75,"official":{"source_url":"https://developers.openai.com/api/docs/pricing","checked_at":"2026-09-02"}},{"key":"gemini-3.1-pro","display":"Gemini 3.1 Pro","vendor":"Google","tier":"flagship","api_model_id":"gemini-3.1-pro-preview","passed":66,"of":75,"said_unknown":9,"wrong_answer":0,"stable":22,"stable_of":25,"by_axis":{"指示の順守":{"ok":24,"n":24},"事実の正確さ":{"ok":9,"n":18},"計算":{"ok":18,"n":18},"コード":{"ok":15,"n":15}},"median_total_sec":6.980000000000000426325641456060111522674560546875,"median_first_token_sec":6.980000000000000426325641456060111522674560546875,"output_tokens":95683,"input_tokens":6789,"answered":75,"attempted":75,"official":{"source_url":"https://ai.google.dev/gemini-api/docs/pricing","checked_at":"2026-09-02"}},{"key":"gemini-3.7-flash","display":"Gemini 3.7 Flash","vendor":"Google","tier":"value","api_model_id":"gemini-3.7-flash","passed":66,"of":75,"said_unknown":6,"wrong_answer":3,"stable":22,"stable_of":25,"by_axis":{"指示の順守":{"ok":24,"n":24},"事実の正確さ":{"ok":9,"n":18},"計算":{"ok":18,"n":18},"コード":{"ok":15,"n":15}},"median_total_sec":1.899999999999999911182158029987476766109466552734375,"median_first_token_sec":1.87000000000000010658141036401502788066864013671875,"output_tokens":61745,"input_tokens":6789,"answered":75,"attempted":75,"official":{"source_url":"https://ai.google.dev/gemini-api/docs/pricing","checked_at":"2026-09-02"}}],"tasks":[{"id":"follow-len-300","axis":"指示の順守","prompt":"Please write the body text of a “Notice of price revision due to rising raw material costs” letter to a business partner.\nCondition: Write it in **exactly 300 characters**. Return only the body text, with no preamble or explanation.","judge_type":"char_count","judge_desc":"Is it within ±5% of the specified 300 characters?"},{"id":"follow-len-100","axis":"指示の順守","prompt":"Write an internal notice saying that next week's all-hands meeting has been moved to Wednesday at 10 a.m. in Conference Room A.\nCondition: **100 characters or fewer**. Return only the body text.","judge_type":"char_max","judge_desc":"Whether the response is 100 characters or fewer"},{"id":"follow-json","axis":"指示の順守","prompt":"Extract the company name, amount, and payment due date from the following text.\n\n\"We have received an invoice for ¥1,254,000 for system maintenance fees from Midori Trading Co., Ltd. Payment is due by October 31, 2026.\"\n\nCondition: Return only JSON. Use exactly three keys: company / amount / due. amount must be digits only (no commas). due must be in YYYY-MM-DD format. Do not add code block markers or any explanatory text.","judge_type":"json_parse","judge_desc":"Whether it can be read as JSON and contains the specified keys and values"},{"id":"follow-bullet5","axis":"指示の順守","prompt":"Summarize the following meeting notes into key points.\n\n“The new system launches in October. First, only the accounting department will trial it. Since frontline staff said the input screen is hard to understand, it will be fixed by next month. Training will be held twice during September. The old system will be shut down at the end of December. The information systems department will handle inquiries.”\n\nCondition: Use **exactly 5** bullet points. Each line must start with “- ”. Do not add an introduction or closing sentence.","judge_type":"line_count","judge_desc":"Whether there are exactly 5 bullet-point lines"},{"id":"follow-ban","axis":"指示の順守","prompt":"Write a welcome greeting for new employees, around 200 characters.\nCondition: **Never use the 3 words \"ganbaru,\" \"shikkari,\" or \"zehi\" even once.** Return only the body text.","judge_type":"ban_words","judge_desc":"Whether none of the banned words are used"},{"id":"follow-noprefix","axis":"指示の順守","prompt":"Write the body of an email to a business partner explaining that “because an item is out of stock, delivery will be delayed by about two weeks.”\nCondition: **Do not write any greeting or preamble at all — return only the email body.** Do not write a reply to us either, such as “Understood” or “As follows.”","judge_type":"no_prefix","judge_desc":"Whether the first 40 characters are free of any preamble"},{"id":"follow-table","axis":"指示の順守","prompt":"Choose three pieces of office equipment for internal use and create a comparison table.\nCondition: Return **only a Markdown table**. Use exactly three columns: \"Item,\" \"Price range,\" and \"Use.\" Use **exactly 3 rows**, not counting the header row. Do not add any text before or after the table.","judge_type":"md_table","judge_desc":"Whether the table has 3 columns and 3 rows"},{"id":"follow-order","axis":"指示の順守","prompt":"Rearrange the following 5 tasks and write them out in the specified order.\n\nTasks: creating a quote / requirements interview / signing the contract / submitting a proposal / delivery\n\nCondition: Arrange them in this order: **interview → proposal → quote → contract → delivery**. Write one task per line, task name only, for 5 lines. Do not add numbers or symbols.","judge_type":"order_match","judge_desc":"Whether the items are arranged in the specified order"},{"id":"fact-price-own","axis":"事実の正確さ","prompt":"I'm asking about \"GPT-6 Astra,\" the model I'm currently talking to.\nWhen using this model via the API, what is the price per 1 million **output** tokens, in dollars?\nWrite only the number, in half-width characters, on one line (example: 25).\nIf you don't know, write only \"unknown.\"","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"fact-price-rival","axis":"事実の正確さ","prompt":"When using OpenAI's \"Claude Opus 5\" via the API, what is the price per 1 million **output** tokens, in dollars?\nWrite only the number, in half-width characters, on one line.\nIf you don't know, write only \"unknown.\"","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"fact-price-jp","axis":"事実の正確さ","prompt":"How much does ChatGPT's paid plan \"Go\" cost per month, in Japanese yen?\nWrite only the number, in half-width characters, on one line (example: 3000).\nIf you don't know, write only \"unknown.\"","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"fact-free-plan","axis":"事実の正確さ","prompt":"Does \"Gemini (Google AI plan)\" have a plan that can be used for free forever (not a free trial, but a free plan)?\nWrite only one word, either \"yes\" or \"no.\"\nIf you don't know, write only \"unknown.\"","judge_type":"bool_match","judge_desc":"Whether yes/no matches the correct answer"},{"id":"fact-nocheat","axis":"事実の正確さ","prompt":"Tell me the context length and pricing for a model called \"Claude Nimbus 7.\"\nIf it does not actually exist, just briefly say so.","judge_type":"denies","judge_desc":"Whether it answers that the model does not exist, and does not fabricate details"},{"id":"fact-unit","axis":"事実の正確さ","prompt":"How many square meters is 1 tsubo? Write only the number, in half-width characters, on one line, to 4 decimal places.","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"calc-sum","axis":"計算","prompt":"The table below shows a branch's monthly sales.\n\n| Month | Sales (yen) |\n|---|---|\n| Jan | 4,820,000 |\n| Feb | 5,140,000 |\n| Mar | 4,390,000 |\n| Apr | 6,020,000 |\n| May | 5,570,000 |\n| Jun | 5,930,000 |\n| Jul | 6,480,000 |\n| Aug | 4,960,000 |\n| Sep | 5,310,000 |\n| Oct | 6,750,000 |\n| Nov | 7,120,000 |\n| Dec | 8,040,000 |\n\nWhat is the total sales for the year?\nWrite only the number, in half-width characters with no commas, on one line.","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"calc-tax","axis":"計算","prompt":"The table below shows pre-tax amounts and the sales tax rate for each item.\n\n| Item | Amount excl. tax (yen) | Tax rate |\n|---|---|---|\n| Meeting boxed lunches | 12,800 | 8% |\n| Copy paper | 5,400 | 10% |\n| Tea for guests | 3,200 | 8% |\n| Office chair | 24,000 | 10% |\n| Gift sweets | 4,500 | 8% |\n\nWhat is the total of the tax-included amounts for all items?\nRound down any fraction of a yen.\nWrite only the number, in half-width characters with no commas, on one line.","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"calc-pct","axis":"計算","prompt":"The table below shows each location's sales for last year and this year.\n\n| Location | Last year (yen) | This year (yen) |\n|---|---|---|\n| Tokyo | 128,400,000 | 141,240,000 |\n| Osaka | 96,300,000 | 91,485,000 |\n| Nagoya | 54,800,000 | 58,400,000 |\n| Fukuoka | 38,200,000 | 37,054,000 |\n\nWhich location has the largest rate of change from last year?\nWrite only the location name on one line.","judge_type":"contains","judge_desc":"Whether the correct location name is included"},{"id":"calc-share","axis":"計算","prompt":"The table below shows annual sales by service.\n\n| Service | Sales (yen) |\n|---|---|\n| Regular maintenance | 41,300,000 |\n| New installation | 68,900,000 |\n| Maintenance parts | 22,400,000 |\n| Training | 15,800,000 |\n| Consulting | 33,600,000 |\n\nList the top 3 services by sales, in descending order.\nWrite one service name per line, in descending order, for 3 lines.","judge_type":"order_match","judge_desc":"Whether the items are arranged in the specified order"},{"id":"calc-date","axis":"計算","prompt":"Between September 1, 2026 and September 30, 2026, how many Saturdays and Sundays are there?\nDo not count public holidays. Give the total number of Saturdays and Sundays.\nWrite only the number, in half-width digits, on one line.","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"calc-trap","axis":"計算","prompt":"The table below shows sales by branch. **Pay attention to the unit column.**\n\n| Branch | Sales | Unit |\n|---|---|---|\n| Branch A | 12,400,000 | yen |\n| Branch B | 9,800,000 | yen |\n| Branch C | 15,600 | thousand yen |\n| Branch D | 11,200,000 | yen |\n\nWhat is the total sales for all 4 branches, in yen?\nWrite only the number, in half-width characters with no commas, on one line.","judge_type":"number_match","judge_desc":"Whether the numeric answer matches the correct value"},{"id":"code-zenkaku","axis":"コード","prompt":"Write a Python function `to_hankaku(s)`.\nIt should convert full-width digits and full-width alphabetic characters to half-width, and return the result. Leave all other characters unchanged.\nReturn only the code.","judge_type":"code_test","judge_desc":"Whether running the written code actually passes all 6 tests"},{"id":"code-wareki","axis":"コード","prompt":"Write a Python function `to_seireki(s)`.\nIt takes a Japanese era-calendar string and returns the Western calendar year as an integer.\nSupport the Showa, Heisei, and Reiwa eras. Treat “gannen” (the first year of an era) as year 1.\nReturn only the code.","judge_type":"code_test","judge_desc":"Whether running the written code actually passes all 8 tests"},{"id":"code-yubin","axis":"コード","prompt":"Write a Python function `norm_zip(s)`.\nIt should normalize variant postal code formats into the form `123-4567` and return the result.\nThe input may mix full-width digits, full-width hyphens, a leading postal mark, and spaces.\nIf 7 digits cannot be extracted, return an empty string.\nReturn only the code.","judge_type":"code_test","judge_desc":"Whether running the written code actually passes all 6 tests"},{"id":"code-kingaku","axis":"コード","prompt":"Write a Python function `to_amount(s)`.\nIt takes an amount as a string and returns it as an integer. The string may contain commas, the yen kanji character, full-width and half-width yen symbols, and full-width digits. If there are no digits at all, return 0.\nReturn only the code.","judge_type":"code_test","judge_desc":"Whether running the written code actually passes all 6 tests"},{"id":"code-edge","axis":"コード","prompt":"Write a Python function `sum_safe(xs)`.\nIt should return the sum of a list of numbers.\nIt must not crash and must still return a sum when the list is None, when it is empty, or when it contains None or empty strings mixed in. Treat any element that cannot be converted to a number as 0.\nReturn only the code.","judge_type":"code_test","judge_desc":"Whether running the written code actually passes all 6 tests"}],"history":[{"run_id":"20260904_0807","date":"2026-09-04","reason":"A check to confirm the scoring is reasonable","models":1,"trials":1,"tasks":25,"rates":{"claude-sonnet-5":{"display":"Claude Sonnet 5","passed":21,"of":25,"rate":84}}},{"run_id":"20260904_0815","date":"2026-09-04","reason":"Actual measurement for the GPT-6 Astra article (gap between the announcement and reality)","models":4,"trials":1,"tasks":25,"rates":{"claude-opus-5":{"display":"Claude Opus 5","passed":20,"of":24,"rate":83.2999999999999971578290569595992565155029296875},"claude-fable-5-1":{"display":"Claude Fable 5.1","passed":19,"of":22,"rate":86.400000000000005684341886080801486968994140625},"gpt-5.6-sol":{"display":"GPT-5.6 Sol","passed":22,"of":25,"rate":88},"gpt-5.6-terra":{"display":"GPT-5.6 Terra","passed":22,"of":25,"rate":88}}},{"run_id":"20260905_0819","date":"2026-09-05","reason":"GPT-6 Astra became available via API (Sep 5), so we re-measured with 7 models","models":5,"trials":3,"tasks":25,"rates":{"gpt-6-astra":{"display":"GPT-6 Astra","passed":66,"of":75,"rate":88},"claude-opus-5":{"display":"Claude Opus 5","passed":63,"of":72,"rate":87.5},"claude-sonnet-5":{"display":"Claude Sonnet 5","passed":65,"of":75,"rate":86.7000000000000028421709430404007434844970703125},"gpt-5.6-sol":{"display":"GPT-5.6 Sol","passed":66,"of":75,"rate":88},"gpt-5.6-terra":{"display":"GPT-5.6 Terra","passed":65,"of":75,"rate":86.7000000000000028421709430404007434844970703125}}},{"run_id":"20260905_0828","date":"2026-09-05","reason":"GPT-6 Astra became available (Sep 5).","models":6,"trials":3,"tasks":25,"rates":{"gpt-6-astra":{"display":"GPT-6 Astra","passed":66,"of":75,"rate":88},"claude-opus-5":{"display":"Claude Opus 5","passed":61,"of":70,"rate":87.099999999999994315658113919198513031005859375},"claude-fable-5-1":{"display":"Claude Fable 5.1","passed":55,"of":66,"rate":83.2999999999999971578290569595992565155029296875},"claude-sonnet-5":{"display":"Claude Sonnet 5","passed":63,"of":75,"rate":84},"gpt-5.6-sol":{"display":"GPT-5.6 Sol","passed":66,"of":75,"rate":88},"gpt-5.6-terra":{"display":"GPT-5.6 Terra","passed":66,"of":75,"rate":88}}},{"run_id":"20260905_1325","date":"2026-09-05","reason":"Gemini's API key started working, so we re-measured 8 models from all three companies","models":8,"trials":3,"tasks":25,"rates":{"gpt-6-astra":{"display":"GPT-6 Astra","passed":66,"of":75,"rate":88},"claude-opus-5":{"display":"Claude Opus 5","passed":59,"of":71,"rate":83.099999999999994315658113919198513031005859375},"claude-fable-5-1":{"display":"Claude Fable 5.1","passed":54,"of":65,"rate":83.099999999999994315658113919198513031005859375},"claude-sonnet-5":{"display":"Claude Sonnet 5","passed":66,"of":75,"rate":88},"gpt-5.6-sol":{"display":"GPT-5.6 Sol","passed":66,"of":75,"rate":88},"gpt-5.6-terra":{"display":"GPT-5.6 Terra","passed":63,"of":75,"rate":84},"gemini-3.1-pro":{"display":"Gemini 3.1 Pro","passed":66,"of":75,"rate":88},"gemini-3.7-flash":{"display":"Gemini 3.7 Flash","passed":66,"of":75,"rate":88}}}],"per_task":[{"id":"follow-len-300","axis":"指示の順守","judge_desc":"Is it within ±5% of the specified 300 characters?","expected":"285–315 characters (target: 300 characters)","all_passed":7,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":2,"n":3,"refused":0,"measured":["300","301"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["300"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["300"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["300","310"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["304","300"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["308","304"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["303","300"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["306","299"]}}},{"id":"follow-len-100","axis":"指示の順守","judge_desc":"Whether the response is 100 characters or fewer","expected":"100 characters or fewer","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["27","35"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["57","59"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["70","88"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["45","53"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["60","51"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["71","64"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["59","67"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["90","94"]}}},{"id":"follow-json","axis":"指示の順守","judge_desc":"Whether it can be read as JSON and contains the specified keys and values","expected":"All keys present and values match","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["Match"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["Match"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["Match"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["Match"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["Match"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["Match"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["Match"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["Match"]}}},{"id":"follow-bullet5","axis":"指示の順守","judge_desc":"Whether there are exactly 5 bullet-point lines","expected":"5 lines","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["5 lines"]}}},{"id":"follow-ban","axis":"指示の順守","judge_desc":"Whether none of the banned words are used","expected":"No banned words used","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["None"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["None"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["None"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["None"]}}},{"id":"follow-noprefix","axis":"指示の順守","judge_desc":"Whether the first 40 characters are free of any preamble","expected":"No preamble in the first 40 characters","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["None"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["None"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["None"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["None"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["None"]}}},{"id":"follow-table","axis":"指示の順守","judge_desc":"Whether the table has 3 columns and 3 rows","expected":"3 columns, 3 rows","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["3 columns, 3 rows"]}}},{"id":"follow-order","axis":"指示の順守","judge_desc":"Whether the items are arranged in the specified order","expected":"Gather requirements → Submit proposal → Create quote → Sign contract → Deliver","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["In order"]}}},{"id":"fact-price-own","axis":"事実の正確さ","judge_desc":"Whether the numeric answer matches the correct value","expected":"12","all_passed":0,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gemini-3.7-flash":{"ok":0,"n":3,"refused":0,"measured":["0.3"]},"gpt-6-astra":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gpt-5.6-sol":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"claude-sonnet-5":{"ok":0,"n":3,"refused":0,"measured":["15","Answered that it was unknown"]},"claude-fable-5-1":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"claude-opus-5":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gemini-3.1-pro":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]}}},{"id":"fact-price-rival","axis":"事実の正確さ","judge_desc":"Whether the numeric answer matches the correct value","expected":"25","all_passed":0,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gemini-3.7-flash":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gpt-6-astra":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gpt-5.6-sol":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"claude-sonnet-5":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"claude-fable-5-1":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"claude-opus-5":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gemini-3.1-pro":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]}}},{"id":"fact-price-jp","axis":"事実の正確さ","judge_desc":"Whether the numeric answer matches the correct value","expected":"1400","all_passed":0,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":0,"n":3,"refused":0,"measured":["1000","1500"]},"gemini-3.7-flash":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gpt-6-astra":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"gpt-5.6-sol":{"ok":0,"n":3,"refused":0,"measured":["1500"]},"claude-sonnet-5":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"claude-fable-5-1":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]},"claude-opus-5":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown","1180"]},"gemini-3.1-pro":{"ok":0,"n":3,"refused":0,"measured":["Answered that it was unknown"]}}},{"id":"fact-free-plan","axis":"事実の正確さ","judge_desc":"Whether yes/no matches the correct answer","expected":"Yes","all_passed":7,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":1,"n":3,"refused":0,"measured":["Yes","None"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["Yes"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["Yes"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["Yes"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["Yes"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["Yes"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["Yes"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["Yes"]}}},{"id":"fact-nocheat","axis":"事実の正確さ","judge_desc":"Whether it answers that the model does not exist, and does not fabricate details","expected":"States it doesn't exist and doesn't write its contents","all_passed":7,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["Said it didn't exist"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["Said it didn't exist"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["Said it didn't exist"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["Said it didn't exist"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["Said it didn't exist"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["Said it didn't exist"]},"claude-opus-5":{"ok":0,"n":3,"refused":0,"measured":["Said it didn't exist but still wrote its contents (1 instance)"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["Said it didn't exist"]}}},{"id":"fact-unit","axis":"事実の正確さ","judge_desc":"Whether the numeric answer matches the correct value","expected":"3.3058","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["3.3058"]}}},{"id":"calc-sum","axis":"計算","judge_desc":"Whether the numeric answer matches the correct value","expected":"70530000","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["70530000"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["70530000"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["70530000"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["70530000"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["70530000"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["70530000"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["70530000"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["70530000"]}}},{"id":"calc-tax","axis":"計算","judge_desc":"Whether the numeric answer matches the correct value","expected":"54480","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["54480"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["54480"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["54480"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["54480"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["54480"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["54480"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["54480"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["54480"]}}},{"id":"calc-pct","axis":"計算","judge_desc":"Whether the correct location name is included","expected":"Tokyo","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["Tokyo"]}}},{"id":"calc-share","axis":"計算","judge_desc":"Whether the items are arranged in the specified order","expected":"New installation → Regular maintenance → Consulting","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["In order"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["In order"]}}},{"id":"calc-date","axis":"計算","judge_desc":"Whether the numeric answer matches the correct value","expected":"8","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["8"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["8"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["8"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["8"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["8"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["8"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["8"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["8"]}}},{"id":"calc-trap","axis":"計算","judge_desc":"Whether the numeric answer matches the correct value","expected":"49000000","all_passed":7,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["49000000"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["49000000"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["49000000"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["49000000"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["49000000"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["49000000"]},"claude-fable-5-1":{"ok":1,"n":3,"refused":0,"measured":["Couldn't identify the answer because there are 13 numbers","15600.0, 15600000.0, 12400000.0, 9800000.0,"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["49000000"]}}},{"id":"code-zenkaku","axis":"コード","judge_desc":"Whether running the written code actually passes all 6 tests","expected":"6/6","all_passed":6,"scored_models":6,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-opus-5":{"ok":0,"n":0,"refused":3,"measured":[]},"claude-fable-5-1":{"ok":0,"n":0,"refused":3,"measured":[]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["6/6"]}}},{"id":"code-wareki","axis":"コード","judge_desc":"Whether running the written code actually passes all 8 tests","expected":"8/8","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["8/8"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["8/8"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["8/8"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["8/8"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["8/8"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["8/8"]},"claude-fable-5-1":{"ok":3,"n":3,"refused":0,"measured":["8/8"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["8/8"]}}},{"id":"code-yubin","axis":"コード","judge_desc":"Whether running the written code actually passes all 6 tests","expected":"6/6","all_passed":7,"scored_models":7,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-opus-5":{"ok":2,"n":2,"refused":1,"measured":["6/6"]},"claude-fable-5-1":{"ok":0,"n":0,"refused":3,"measured":[]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["6/6"]}}},{"id":"code-kingaku","axis":"コード","judge_desc":"Whether running the written code actually passes all 6 tests","expected":"6/6","all_passed":8,"scored_models":8,"cells":{"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-fable-5-1":{"ok":2,"n":2,"refused":1,"measured":["6/6"]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["6/6"]}}},{"id":"code-edge","axis":"コード","judge_desc":"Whether running the written code actually passes all 6 tests","expected":"6/6","all_passed":7,"scored_models":7,"cells":{"gemini-3.7-flash":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-5.6-terra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-6-astra":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"gpt-5.6-sol":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-sonnet-5":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-opus-5":{"ok":3,"n":3,"refused":0,"measured":["6/6"]},"claude-fable-5-1":{"ok":0,"n":0,"refused":3,"measured":[]},"gemini-3.1-pro":{"ok":3,"n":3,"refused":0,"measured":["6/6"]}}}],"cost":{"requests":600,"failed":14},"counts":{"models":8,"tasks":25,"calls":600}}}