{ "schemaVersion": 1, "retrievedAt": "2026-10-25T04:34:11Z", "buildDate": "2026-10-25", "scoringPolicy": "Separate publisher scales; no cross-source aggregate, approximate model aliases or missing-to-zero coercion", "sources": [ { "id": "livebench2", "label": "LiveBench 2.0", "url": "https://livebench.ai/", "methodologyUrl": "https://livebench.ai/#/methodology", "asOf": "Release 2026-10-14; publisher update 23 October 2026", "retrievedAt": "2026-10-25T04:30:37Z", "description": "Seven task suites on the 0–100 LiveBench scale. Overall follows the publisher's mean of available suites; inspect individual suite scores using the metric selector.", "coverage": "53 exact variants; 7 suites; 371 published suite cells", "caveats": [ "Each cell is the model's mean task score for the suite, 0–100, from a single run. A task the run could not measure, even after one rerun, scores 0.", "Design: layout and design tasks across presentations, PDFs and documents. Graded on visual and structural checks of the produced file.", "Research: open-ended research briefs producing a written report. Graded against a rubric.", "Browser Use: multi-step tasks in working web applications, completed through a real browser. Scored as the share of tasks passed. A model that cannot accept screenshots cannot do these tasks and scores 0.", "Coding: bug-fix tasks taken from real merged pull requests on a frozen codebase. Deterministic grading; a fix that breaks existing behaviour scores 0.", "Agentic: long-running work — applications extended over several turns of changing requirements, operations on an existing codebase over simulated months or weeks, and extended coding and optimisation tasks. Deterministic grading.", "Data Analysis: short data-analysis questions on small tables — finding runs of consecutive growth, mapping the columns of one table onto another, and converting a table between formats — each with a single correct answer, graded against ground truth. Each question costs far less than an agentic task.", "Reasoning: short logic, spatial and theory-of-mind questions, each with a single correct answer, graded against ground truth. Each question costs far less than an agentic task.", "Models run at their highest supported effort setting. Where a different setting was used, it is shown in the model's expanded row.", "Overall is the mean of the suites a model has a score for. Blank and n/a cells are left out. Every suite counts equally.", "This version is not comparable to the historical LiveBench 1.0 table.", "Overall is derived according to the publisher rule, not an extra independent evaluation.", "Exact evaluation dates and statistical intervals are not published in this snapshot." ], "metrics": [ { "key": "overall", "label": "Overall", "unit": "0–100" }, { "key": "Long Agentic", "label": "Long Agentic", "unit": "0–100" }, { "key": "Reasoning", "label": "Reasoning", "unit": "0–100" }, { "key": "Coding", "label": "Coding", "unit": "0–100" }, { "key": "Browser", "label": "Browser", "unit": "0–100" }, { "key": "Data Analysis", "label": "Data Analysis", "unit": "0–100" }, { "key": "Research", "label": "Research", "unit": "0–100" }, { "key": "Design", "label": "Design", "unit": "0–100" } ], "rows": [ { "id": "beam-1.0", "model": "Beam 1.0", "org": "Reflection AI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://reflection.ai/", "metrics": { "overall": 55.5, "Long Agentic": 17.3, "Reasoning": 80.2, "Coding": 83.9, "Browser": 0.0, "Data Analysis": 66.4, "Research": 72.8, "Design": 67.9 }, "details": { "Publisher model ID": "beam-1.0", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-fable-5.1", "model": "Claude Fable 5.1", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/news/claude-fable-5-1", "metrics": { "overall": 87.99, "Long Agentic": 80.8, "Reasoning": 91.7, "Coding": 97.7, "Browser": 100.0, "Data Analysis": 80.3, "Research": 82.0, "Design": 83.4 }, "details": { "Publisher model ID": "claude-fable-5.1", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-fable-5", "model": "Claude Fable 5", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/news/claude-fable-5", "metrics": { "overall": 83.61, "Long Agentic": 53.2, "Reasoning": 89.7, "Coding": 98.3, "Browser": 92.0, "Data Analysis": 80.5, "Research": 86.9, "Design": 84.7 }, "details": { "Publisher model ID": "claude-fable-5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-haiku-4.5", "model": "Claude Haiku 4.5", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/news/claude-haiku-4-5", "metrics": { "overall": 46.04, "Long Agentic": 11.1, "Reasoning": 61.7, "Coding": 48.1, "Browser": 40.0, "Data Analysis": 59.3, "Research": 42.2, "Design": 59.9 }, "details": { "Publisher model ID": "claude-haiku-4.5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-haiku-5.5", "model": "Claude Haiku 5.5", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.claude.com/docs/en/models/haiku-5-5/overview", "metrics": { "overall": 75.64, "Long Agentic": 57.6, "Reasoning": 82.4, "Coding": 91.4, "Browser": 80.0, "Data Analysis": 53.1, "Research": 81.9, "Design": 83.1 }, "details": { "Publisher model ID": "claude-haiku-5.5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-opus-4.8", "model": "Claude Opus 4.8", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/news/claude-opus-4-8", "metrics": { "overall": 76.77, "Long Agentic": 48.3, "Reasoning": 89.2, "Coding": 77.3, "Browser": 96.0, "Data Analysis": 66.0, "Research": 81.8, "Design": 78.8 }, "details": { "Publisher model ID": "claude-opus-4.8", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-opus-5.5", "model": "Claude Opus 5.5", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/claude/opus", "metrics": { "overall": 86.53, "Long Agentic": 85.0, "Reasoning": 92.2, "Coding": 96.3, "Browser": 80.0, "Data Analysis": 80.3, "Research": 85.0, "Design": 86.9 }, "details": { "Publisher model ID": "claude-opus-5.5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-opus-5", "model": "Claude Opus 5", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/news/claude-opus-5", "metrics": { "overall": 82.09, "Long Agentic": 62.1, "Reasoning": 91.2, "Coding": 90.8, "Browser": 84.0, "Data Analysis": 74.6, "Research": 88.9, "Design": 83.0 }, "details": { "Publisher model ID": "claude-opus-5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-sonnet-4.6", "model": "Claude Sonnet 4.6", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/news/claude-sonnet-4-6", "metrics": { "overall": 65.09, "Long Agentic": 23.1, "Reasoning": 86.4, "Coding": 66.6, "Browser": 68.0, "Data Analysis": 76.1, "Research": 63.6, "Design": 71.8 }, "details": { "Publisher model ID": "claude-sonnet-4.6", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-sonnet-5.5", "model": "Claude Sonnet 5.5", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/claude/sonnet", "metrics": { "overall": 82.46, "Long Agentic": 78.8, "Reasoning": 91.6, "Coding": 94.7, "Browser": 84.0, "Data Analysis": 59.5, "Research": 83.6, "Design": 85.0 }, "details": { "Publisher model ID": "claude-sonnet-5.5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "claude-sonnet-5", "model": "Claude Sonnet 5", "org": "Anthropic", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.anthropic.com/news/claude-sonnet-5", "metrics": { "overall": 72.99, "Long Agentic": 39.0, "Reasoning": 88.7, "Coding": 74.7, "Browser": 72.0, "Data Analysis": 71.7, "Research": 82.8, "Design": 82.0 }, "details": { "Publisher model ID": "claude-sonnet-5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "deepseek-v4-flash-vision", "model": "DeepSeek V4 Flash Vision", "org": "DeepSeek", "settings": "Effort: max; version: exp", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://api-docs.deepseek.com/updates", "metrics": { "overall": 63.63, "Long Agentic": 39.0, "Reasoning": 85.4, "Coding": 37.6, "Browser": 64.0, "Data Analysis": 79.5, "Research": 69.2, "Design": 70.7 }, "details": { "Publisher model ID": "deepseek-v4-flash-vision", "Explicit version": "exp", "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "deepseek-v4-flash", "model": "DeepSeek V4 Flash 0731", "org": "DeepSeek", "settings": "Effort: max; version: 0731", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://api-docs.deepseek.com/quick_start/pricing/", "metrics": { "overall": 57.59, "Long Agentic": 36.6, "Reasoning": 70.6, "Coding": 77.0, "Browser": 0.0, "Data Analysis": 68.0, "Research": 79.2, "Design": 71.7 }, "details": { "Publisher model ID": "deepseek-v4-flash", "Explicit version": "0731", "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "deepseek-v4.1-flash-1017", "model": "DeepSeek V4.1 Flash 1017", "org": "DeepSeek", "settings": "Effort: max; version: 1017", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://api-docs.deepseek.com/updates", "metrics": { "overall": 79.2, "Long Agentic": 50.9, "Reasoning": 89.1, "Coding": 96.0, "Browser": 76.0, "Data Analysis": 79.5, "Research": 79.5, "Design": 83.4 }, "details": { "Publisher model ID": "deepseek-v4.1-flash-1017", "Explicit version": "1017", "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "deepseek-v4.1-flash", "model": "DeepSeek V4.1 Flash", "org": "DeepSeek", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://api-docs.deepseek.com/updates", "metrics": { "overall": 78.04, "Long Agentic": 40.1, "Reasoning": 86.7, "Coding": 90.2, "Browser": 84.0, "Data Analysis": 79.3, "Research": 86.8, "Design": 79.2 }, "details": { "Publisher model ID": "deepseek-v4.1-flash", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gemini-3.1-pro", "model": "Gemini 3.1 Pro", "org": "Google", "settings": "Effort: high; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://ai.google.dev/gemini-api/docs/models/gemini-3.1-pro-preview/", "metrics": { "overall": 54.91, "Long Agentic": 23.9, "Reasoning": 84.0, "Coding": 66.7, "Browser": 4.0, "Data Analysis": 78.5, "Research": 65.7, "Design": 61.6 }, "details": { "Publisher model ID": "gemini-3.1-pro", "Explicit version": null, "Effort": "high", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gemini-3.7-flash", "model": "Gemini 3.7 Flash", "org": "Google", "settings": "Effort: high; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://ai.google.dev/gemini-api/docs/models/gemini-3.7-flash", "metrics": { "overall": 67.19, "Long Agentic": 28.3, "Reasoning": 87.8, "Coding": 80.5, "Browser": 56.0, "Data Analysis": 68.0, "Research": 76.5, "Design": 73.2 }, "details": { "Publisher model ID": "gemini-3.7-flash", "Explicit version": null, "Effort": "high", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gemini-3.8-flash", "model": "Gemini 3.8 Flash", "org": "Google", "settings": "Effort: high; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://ai.google.dev/gemini-api/docs/models/gemini-3.8-flash", "metrics": { "overall": 67.34, "Long Agentic": 30.1, "Reasoning": 89.3, "Coding": 82.2, "Browser": 64.0, "Data Analysis": 54.0, "Research": 79.1, "Design": 72.7 }, "details": { "Publisher model ID": "gemini-3.8-flash", "Explicit version": null, "Effort": "high", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gemini-4-argon", "model": "Gemini 4 Argon", "org": "Google", "settings": "Effort: high; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://blog.google/innovation-and-ai/models-and-research/gemini-models/gemini-4-argon/", "metrics": { "overall": 84.1, "Long Agentic": 69.9, "Reasoning": 95.1, "Coding": 89.7, "Browser": 92.0, "Data Analysis": 85.2, "Research": 83.1, "Design": 73.7 }, "details": { "Publisher model ID": "gemini-4-argon", "Explicit version": null, "Effort": "high", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gemma-4-31b", "model": "Gemma 4 31B", "org": "Google", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://blog.google/innovation-and-ai/technology/developers-tools/gemma-4/", "metrics": { "overall": 41.53, "Long Agentic": 13.3, "Reasoning": 68.3, "Coding": 49.0, "Browser": 4.0, "Data Analysis": 53.3, "Research": 44.4, "Design": 58.4 }, "details": { "Publisher model ID": "gemma-4-31b", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "glm-5.3-flash", "model": "GLM-5.3 Flash", "org": "Z.AI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://z.ai", "metrics": { "overall": 70.61, "Long Agentic": 28.4, "Reasoning": 77.6, "Coding": 80.6, "Browser": 84.0, "Data Analysis": 76.4, "Research": 70.1, "Design": 77.2 }, "details": { "Publisher model ID": "glm-5.3-flash", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "glm-5.3", "model": "GLM-5.3", "org": "Z.AI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://z.ai", "metrics": { "overall": 60.74, "Long Agentic": 35.9, "Reasoning": 85.8, "Coding": 82.8, "Browser": 0.0, "Data Analysis": 70.2, "Research": 73.2, "Design": 77.3 }, "details": { "Publisher model ID": "glm-5.3", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-5.5", "model": "GPT-5.5", "org": "OpenAI", "settings": "Effort: xhigh; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-5.5", "metrics": { "overall": 68.7, "Long Agentic": 44.6, "Reasoning": 89.7, "Coding": 85.1, "Browser": 48.0, "Data Analysis": 81.6, "Research": 61.6, "Design": 70.3 }, "details": { "Publisher model ID": "gpt-5.5", "Explicit version": null, "Effort": "xhigh", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-5.6-luna", "model": "GPT-5.6 Luna", "org": "OpenAI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-5.6", "metrics": { "overall": 66.09, "Long Agentic": 30.4, "Reasoning": 85.6, "Coding": 73.6, "Browser": 68.0, "Data Analysis": 78.0, "Research": 51.0, "Design": 76.0 }, "details": { "Publisher model ID": "gpt-5.6-luna", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-5.6-sol", "model": "GPT-5.6 Sol", "org": "OpenAI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-5.6", "metrics": { "overall": 76.26, "Long Agentic": 48.5, "Reasoning": 91.7, "Coding": 89.7, "Browser": 88.0, "Data Analysis": 79.8, "Research": 54.5, "Design": 81.6 }, "details": { "Publisher model ID": "gpt-5.6-sol", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-5.6-terra", "model": "GPT-5.6 Terra", "org": "OpenAI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-5.6", "metrics": { "overall": 74.99, "Long Agentic": 48.4, "Reasoning": 90.6, "Coding": 86.2, "Browser": 84.0, "Data Analysis": 79.3, "Research": 60.4, "Design": 76.0 }, "details": { "Publisher model ID": "gpt-5.6-terra", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-6-astra", "model": "GPT-6 Astra", "org": "OpenAI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-6", "metrics": { "overall": 86.14, "Long Agentic": 82.4, "Reasoning": 92.7, "Coding": 93.7, "Browser": 84.0, "Data Analysis": 83.0, "Research": 85.2, "Design": 82.0 }, "details": { "Publisher model ID": "gpt-6-astra", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-6-luna", "model": "GPT-6 Luna", "org": "OpenAI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-6", "metrics": { "overall": 69.4, "Long Agentic": 48.1, "Reasoning": 81.8, "Coding": 84.5, "Browser": 72.0, "Data Analysis": 73.4, "Research": 43.5, "Design": 82.5 }, "details": { "Publisher model ID": "gpt-6-luna", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-6-sol", "model": "GPT-6 Sol", "org": "OpenAI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-6", "metrics": { "overall": 81.91, "Long Agentic": 66.9, "Reasoning": 88.7, "Coding": 93.1, "Browser": 92.0, "Data Analysis": 81.2, "Research": 69.3, "Design": 82.2 }, "details": { "Publisher model ID": "gpt-6-sol", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "gpt-6.1-sol", "model": "GPT-6.1 Sol", "org": "OpenAI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.openai.com/docs/models/gpt-6", "metrics": { "overall": 85.41, "Long Agentic": 73.0, "Reasoning": 92.6, "Coding": 96.0, "Browser": 88.0, "Data Analysis": 82.7, "Research": 83.5, "Design": 82.1 }, "details": { "Publisher model ID": "gpt-6.1-sol", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "grok-4.5", "model": "Grok 4.5", "org": "xAI", "settings": "Effort: xhigh; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://docs.x.ai/developers/models/grok-4.5", "metrics": { "overall": 70.06, "Long Agentic": 27.9, "Reasoning": 87.2, "Coding": 94.3, "Browser": 44.0, "Data Analysis": 73.0, "Research": 85.9, "Design": 78.1 }, "details": { "Publisher model ID": "grok-4.5", "Explicit version": null, "Effort": "xhigh", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "grok-4.6", "model": "Grok 4.6", "org": "xAI", "settings": "Effort: xhigh; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://x.ai/news/grok-4-6", "metrics": { "overall": 74.94, "Long Agentic": 40.7, "Reasoning": 91.2, "Coding": 89.1, "Browser": 64.0, "Data Analysis": 75.3, "Research": 84.5, "Design": 79.8 }, "details": { "Publisher model ID": "grok-4.6", "Explicit version": null, "Effort": "xhigh", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "grok-4.7", "model": "Grok 4.7", "org": "xAI", "settings": "Effort: xhigh; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://docs.x.ai/developers/models/grok-4.7", "metrics": { "overall": 74.09, "Long Agentic": 50.1, "Reasoning": 82.7, "Coding": 84.5, "Browser": 60.0, "Data Analysis": 76.9, "Research": 86.4, "Design": 78.0 }, "details": { "Publisher model ID": "grok-4.7", "Explicit version": null, "Effort": "xhigh", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "inkling", "model": "Inkling", "org": "Thinking Machines", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://thinkingmachines.ai/", "metrics": { "overall": 51.91, "Long Agentic": 14.1, "Reasoning": 78.3, "Coding": 64.0, "Browser": 16.0, "Data Analysis": 72.8, "Research": 59.9, "Design": 58.3 }, "details": { "Publisher model ID": "inkling", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "kimi-k3", "model": "Kimi K3", "org": "Moonshot AI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.kimi.com/blog/kimi-k3", "metrics": { "overall": 63.39, "Long Agentic": 42.8, "Reasoning": 90.7, "Coding": 59.3, "Browser": 16.0, "Data Analysis": 78.7, "Research": 79.0, "Design": 77.2 }, "details": { "Publisher model ID": "kimi-k3", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "mimo-v2.6-pro", "model": "MiMo V2.6 Pro", "org": "Xiaomi", "settings": "Effort: high; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://mimo.xiaomi.com/", "metrics": { "overall": 74.87, "Long Agentic": 47.5, "Reasoning": 79.6, "Coding": 81.5, "Browser": 88.0, "Data Analysis": 72.3, "Research": 76.7, "Design": 78.5 }, "details": { "Publisher model ID": "mimo-v2.6-pro", "Explicit version": null, "Effort": "high", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "minimax-m3.1-flash", "model": "MiniMax M3.1 Flash", "org": "MiniMax", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://platform.minimax.io/docs/guides/text-generation", "metrics": { "overall": 73.09, "Long Agentic": 34.7, "Reasoning": 82.9, "Coding": 83.9, "Browser": 92.0, "Data Analysis": 71.4, "Research": 72.7, "Design": 74.0 }, "details": { "Publisher model ID": "minimax-m3.1-flash", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "minimax-m3", "model": "MiniMax M3", "org": "MiniMax", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.minimax.io/", "metrics": { "overall": 53.74, "Long Agentic": 12.4, "Reasoning": 74.5, "Coding": 70.9, "Browser": 8.0, "Data Analysis": 76.2, "Research": 68.0, "Design": 66.2 }, "details": { "Publisher model ID": "minimax-m3", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "mistral-large-4", "model": "Mistral Large 4", "org": "Mistral AI", "settings": "Effort: high; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://mistral.ai/", "metrics": { "overall": 47.7, "Long Agentic": 11.8, "Reasoning": 83.9, "Coding": 60.0, "Browser": 8.0, "Data Analysis": 76.5, "Research": 28.1, "Design": 65.6 }, "details": { "Publisher model ID": "mistral-large-4", "Explicit version": null, "Effort": "high", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "muse-spark-1.2", "model": "Muse Spark 1.2", "org": "Meta", "settings": "Effort: xhigh; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://ai.meta.com/", "metrics": { "overall": 62.4, "Long Agentic": 21.3, "Reasoning": 90.0, "Coding": 78.2, "Browser": 36.0, "Data Analysis": 76.5, "Research": 67.7, "Design": 67.1 }, "details": { "Publisher model ID": "muse-spark-1.2", "Explicit version": null, "Effort": "xhigh", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "muse-spark-1.3", "model": "Muse Spark 1.3", "org": "Meta", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://ai.meta.com/", "metrics": { "overall": 74.76, "Long Agentic": 40.4, "Reasoning": 89.7, "Coding": 90.2, "Browser": 80.0, "Data Analysis": 79.6, "Research": 68.6, "Design": 74.8 }, "details": { "Publisher model ID": "muse-spark-1.3", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "muse-spark-open", "model": "Muse Spark Open", "org": "Meta", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://ai.meta.com/", "metrics": { "overall": 57.57, "Long Agentic": 18.9, "Reasoning": 73.7, "Coding": 73.6, "Browser": 36.0, "Data Analysis": 60.2, "Research": 80.3, "Design": 60.3 }, "details": { "Publisher model ID": "muse-spark-open", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "muse-wonder-1.0", "model": "Muse Wonder 1.0", "org": "Meta", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://ai.meta.com/", "metrics": { "overall": 78.1, "Long Agentic": 53.2, "Reasoning": 90.6, "Coding": 86.2, "Browser": 72.0, "Data Analysis": 81.6, "Research": 88.0, "Design": 75.1 }, "details": { "Publisher model ID": "muse-wonder-1.0", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "nemotron-lightning-3.5", "model": "Nemotron 3.5 Lightning", "org": "NVIDIA", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "metrics": { "overall": 28.69, "Long Agentic": 8.3, "Reasoning": 42.9, "Coding": 22.6, "Browser": 0.0, "Data Analysis": 43.1, "Research": 40.0, "Design": 43.9 }, "details": { "Publisher model ID": "nemotron-lightning-3.5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "nemotron-super-vl-3.5-nvfp4", "model": "Nemotron 3.5 Super VL (NVFP4)", "org": "NVIDIA", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "metrics": { "overall": 50.03, "Long Agentic": 11.5, "Reasoning": 71.7, "Coding": 61.3, "Browser": 12.0, "Data Analysis": 65.7, "Research": 70.9, "Design": 57.1 }, "details": { "Publisher model ID": "nemotron-super-vl-3.5-nvfp4", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "nemotron-super-vl-3.5", "model": "Nemotron 3.5 Super VL", "org": "NVIDIA", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "metrics": { "overall": 51.47, "Long Agentic": 15.8, "Reasoning": 78.8, "Coding": 65.5, "Browser": 4.0, "Data Analysis": 64.6, "Research": 70.9, "Design": 60.7 }, "details": { "Publisher model ID": "nemotron-super-vl-3.5", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "nemotron-ultra-3", "model": "Nemotron 3 Ultra", "org": "NVIDIA", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "metrics": { "overall": 40.83, "Long Agentic": 11.2, "Reasoning": 74.7, "Coding": 43.6, "Browser": 0.0, "Data Analysis": 54.5, "Research": 43.2, "Design": 58.6 }, "details": { "Publisher model ID": "nemotron-ultra-3", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "qwen3.8-27b", "model": "Qwen3.8 27B", "org": "Alibaba", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.qwencloud.com/models/qwen3.8-27b", "metrics": { "overall": 62.66, "Long Agentic": 27.9, "Reasoning": 80.0, "Coding": 86.8, "Browser": 32.0, "Data Analysis": 76.6, "Research": 59.4, "Design": 75.9 }, "details": { "Publisher model ID": "qwen3.8-27b", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "qwen3.8-flash", "model": "Qwen3.8 Flash", "org": "Alibaba", "settings": "Effort: max; version: next", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://qwen.ai/blog?id=qwen3.8", "metrics": { "overall": 68.79, "Long Agentic": 37.1, "Reasoning": 87.4, "Coding": 87.4, "Browser": 48.0, "Data Analysis": 74.2, "Research": 74.6, "Design": 72.8 }, "details": { "Publisher model ID": "qwen3.8-flash", "Explicit version": "next", "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "qwen3.8-max", "model": "Qwen3.8 Max", "org": "Alibaba", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://qwen.ai/blog?id=qwen3.8", "metrics": { "overall": 64.24, "Long Agentic": 42.3, "Reasoning": 88.2, "Coding": 60.3, "Browser": 32.0, "Data Analysis": 78.4, "Research": 69.9, "Design": 78.6 }, "details": { "Publisher model ID": "qwen3.8-max", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "smaug-nemotron-3.5-nvfp4", "model": "Smaug-Nemotron-3.5-NVFP4", "org": "Abacus.AI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/", "metrics": { "overall": 52.93, "Long Agentic": 19.6, "Reasoning": 75.1, "Coding": 72.2, "Browser": 8.0, "Data Analysis": 65.1, "Research": 67.8, "Design": 62.7 }, "details": { "Publisher model ID": "smaug-nemotron-3.5-nvfp4", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "step-5-preview", "model": "Step 5 (Preview)", "org": "StepFun", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.stepfun.com/", "metrics": { "overall": 62.01, "Long Agentic": 27.5, "Reasoning": 83.8, "Coding": 85.6, "Browser": 24.0, "Data Analysis": 63.5, "Research": 72.8, "Design": 76.9 }, "details": { "Publisher model ID": "step-5-preview", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } }, { "id": "kimi-k3.1-pro", "model": "Kimi K3.1 Pro", "org": "Moonshot AI", "settings": "Effort: max; version: as identified by source slug", "date": null, "estimated": false, "status": "Independent · single run", "url": "https://www.kimi.com/en/blog/kimi-k3-1-pro", "metrics": { "overall": 76.74, "Long Agentic": 49.5, "Reasoning": 91.1, "Coding": 81.8, "Browser": 80.0, "Data Analysis": 79.1, "Research": 77.9, "Design": 77.8 }, "details": { "Publisher model ID": "kimi-k3.1-pro", "Explicit version": null, "Effort": "max", "Suite coverage": { "measured_suites": 7, "total_suites": 7 }, "Cell states": { "long_agentic": "measured", "reasoning": "measured", "coding": "measured", "browser": "measured", "data_analysis": "measured", "research": "measured", "design": "measured" }, "Uncertainty": "Single run; no publisher confidence interval", "Raw data": "https://livebench.ai/data/20261014/table.csv" } } ] }, { "id": "aa", "label": "Artificial Analysis · Intelligence", "url": "https://artificialanalysis.ai/leaderboards/models", "methodologyUrl": "https://artificialanalysis.ai/methodology/intelligence-benchmarking", "asOf": "Intelligence Index v4.3.2; evaluation dates not published", "retrievedAt": "2026-10-25T04:31:33Z", "description": "Artificial Analysis Intelligence Index v4.3.2. Exact effort and fallback variants retained. Publisher estimates are hidden by default. SciCode is under review; see the methodology note below.", "coverage": { "models_in_public_current_table": 272, "non_estimated_scored_models": 132, "estimated_scored_models": 134, "missing_index_models": 6 }, "caveats": [ "Retrieval time and HTTP Date are not individual evaluation dates; the public page does not disclose those dates.", "Index points are not percent accuracy. Do not average raw index points with percentages, Elo or ECI.", "SciCode is under review after a possible dataset-error audit: https://artificialanalysis.ai/evaluations/scicode", "The index and its components are not counted as independent measurements of the same evidence.", "Missing scores stay unavailable. Publisher-estimated indices are explicitly marked.", "Pricing and speed in row details are publisher snapshot values, not a purchase quote; deployment providers can differ." ], "metrics": [ { "key": "index", "label": "Intelligence Index", "unit": "index points" } ], "rows": [ { "id": "aa:claude-opus-5-5", "model": "Claude Opus 5.5 (max with fallback)", "org": "Anthropic", "settings": "Max, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-opus-5-5", "metrics": { "index": 57.6223698102963 }, "details": { "Publisher model slug": "claude-opus-5-5", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 5.982012019521066, "Input price USD / million tokens": 4, "Output price USD / million tokens": 20, "Median output tokens / second": 96.2932485108269, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-sonnet-5-5", "model": "Claude Sonnet 5.5 (max with fallback)", "org": "Anthropic", "settings": "Max, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-sonnet-5-5", "metrics": { "index": 56.0001286580794 }, "details": { "Publisher model slug": "claude-sonnet-5-5", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 5.4608525948098325, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 132.7139358127, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-opus-5-5-xhigh", "model": "Claude Opus 5.5 (xhigh with fallback)", "org": "Anthropic", "settings": "Xhigh, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-opus-5-5-xhigh", "metrics": { "index": 55.9873505840139 }, "details": { "Publisher model slug": "claude-opus-5-5-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 3.459110175822289, "Input price USD / million tokens": 4, "Output price USD / million tokens": 20, "Median output tokens / second": 84.8488629341304, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:muse-wonder-full", "model": "Muse Wonder Full (max)", "org": "Meta", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/muse-wonder-full", "metrics": { "index": 55.8021199855616 }, "details": { "Publisher model slug": "muse-wonder-full", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:kimi-k3-1-pro", "model": "Kimi K3.1 Pro", "org": "Kimi", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/kimi-k3-1-pro", "metrics": { "index": 55.0869210959395 }, "details": { "Publisher model slug": "kimi-k3-1-pro", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 2.1145052918767275, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 73.4597669926874, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-opus-5-5-high", "model": "Claude Opus 5.5 (high with fallback)", "org": "Anthropic", "settings": "High, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-opus-5-5-high", "metrics": { "index": 53.5831959822067 }, "details": { "Publisher model slug": "claude-opus-5-5-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.822504718771704, "Input price USD / million tokens": 4, "Output price USD / million tokens": 20, "Median output tokens / second": 80.6496900435159, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-fable-5-1", "model": "Claude Fable 5.1 (max with fallback)", "org": "Anthropic", "settings": "Max, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-fable-5-1", "metrics": { "index": 53.3549259623252 }, "details": { "Publisher model slug": "claude-fable-5-1", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 7.629706364004841, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 68.4576696869576, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-fable-5-1-xhigh", "model": "Claude Fable 5.1 (xhigh with fallback)", "org": "Anthropic", "settings": "Xhigh, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-fable-5-1-xhigh", "metrics": { "index": 53.2032587538944 }, "details": { "Publisher model slug": "claude-fable-5-1-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 5.978304247388257, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 64.7446944201376, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-astra", "model": "GPT-6 Astra (max)", "org": "OpenAI", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-astra", "metrics": { "index": 52.673669395513 }, "details": { "Publisher model slug": "gpt-6-astra", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 3.2575003134834164, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 58.1181143902355, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemini-4-argon", "model": "Gemini 4 Argon (high)", "org": "Google", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemini-4-argon", "metrics": { "index": 52.5605655745982 }, "details": { "Publisher model slug": "gemini-4-argon", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.990322379299008, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 79.1087598186695, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:muse-wonder", "model": "Muse Wonder (max)", "org": "Meta", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/muse-wonder", "metrics": { "index": 52.393894697293 }, "details": { "Publisher model slug": "muse-wonder", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1048576, "Cost per index task (USD)": 4.540034681803926, "Input price USD / million tokens": 4, "Output price USD / million tokens": 20, "Median output tokens / second": 145.174852588055, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-astra-xhigh", "model": "GPT-6 Astra (xhigh)", "org": "OpenAI", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-astra-xhigh", "metrics": { "index": 52.3863277108782 }, "details": { "Publisher model slug": "gpt-6-astra-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 2.308795912269076, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 54.2712698478339, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-sonnet-5-5-xhigh", "model": "Claude Sonnet 5.5 (xhigh with fallback)", "org": "Anthropic", "settings": "Xhigh, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-sonnet-5-5-xhigh", "metrics": { "index": 51.8955544183089 }, "details": { "Publisher model slug": "claude-sonnet-5-5-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 2.0120345801545785, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 111.09564526252, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-1-sol", "model": "GPT-6.1 Sol (max)", "org": "OpenAI", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-1-sol", "metrics": { "index": 51.8332597011541 }, "details": { "Publisher model slug": "gpt-6-1-sol", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.7241670655535033, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 57.3369712661238, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-opus-5-5-medium", "model": "Claude Opus 5.5 (medium with fallback)", "org": "Anthropic", "settings": "Medium, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-opus-5-5-medium", "metrics": { "index": 51.2434931792768 }, "details": { "Publisher model slug": "claude-opus-5-5-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.3360093438976588, "Input price USD / million tokens": 4, "Output price USD / million tokens": 20, "Median output tokens / second": 83.6712981968725, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-fable-5-1-high", "model": "Claude Fable 5.1 (high with fallback)", "org": "Anthropic", "settings": "High, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-fable-5-1-high", "metrics": { "index": 51.1515708852237 }, "details": { "Publisher model slug": "claude-fable-5-1-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 3.912526554325558, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 59.5717841709424, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-1-sol-xhigh", "model": "GPT-6.1 Sol (xhigh)", "org": "OpenAI", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-1-sol-xhigh", "metrics": { "index": 51.0377679093761 }, "details": { "Publisher model slug": "gpt-6-1-sol-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.39286117158850653, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 51.3415212182558, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-astra-high", "model": "GPT-6 Astra (high)", "org": "OpenAI", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-astra-high", "metrics": { "index": 50.9191471636152 }, "details": { "Publisher model slug": "gpt-6-astra-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.7252530861048456, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 56.6147429977635, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-1-sol-high", "model": "GPT-6.1 Sol (high)", "org": "OpenAI", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-1-sol-high", "metrics": { "index": 50.2377777519769 }, "details": { "Publisher model slug": "gpt-6-1-sol-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.31914442375664703, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 50.7375123622524, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-astra-medium", "model": "GPT-6 Astra (medium)", "org": "OpenAI", "settings": "Medium", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-astra-medium", "metrics": { "index": 49.5704363034369 }, "details": { "Publisher model slug": "gpt-6-astra-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.5406493220021167, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 55.2053206449788, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-fable-5-1-medium", "model": "Claude Fable 5.1 (medium with fallback)", "org": "Anthropic", "settings": "Medium, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-fable-5-1-medium", "metrics": { "index": 48.9206162609523 }, "details": { "Publisher model slug": "claude-fable-5-1-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 2.982597436096585, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 58.6268622247708, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-1-sol-medium", "model": "GPT-6.1 Sol (medium)", "org": "OpenAI", "settings": "Medium", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-1-sol-medium", "metrics": { "index": 47.7833271274065 }, "details": { "Publisher model slug": "gpt-6-1-sol-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.21370088267875537, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 48.6199012668899, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-fable-5-1-low", "model": "Claude Fable 5.1 (low with fallback)", "org": "Anthropic", "settings": "Low, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-fable-5-1-low", "metrics": { "index": 46.8163422014862 }, "details": { "Publisher model slug": "claude-fable-5-1-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 2.371025320183763, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 59.9942516568993, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-sonnet-5-5-high", "model": "Claude Sonnet 5.5 (high with fallback)", "org": "Anthropic", "settings": "High, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-sonnet-5-5-high", "metrics": { "index": 46.7533528066958 }, "details": { "Publisher model slug": "claude-sonnet-5-5-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.8844498581005511, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 107.852008795003, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:grok-4-7", "model": "Grok 4.7 (xhigh)", "org": "SpaceXAI", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/grok-4-7", "metrics": { "index": 46.4465506302286 }, "details": { "Publisher model slug": "grok-4-7", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 500000, "Cost per index task (USD)": 3.738325952106939, "Input price USD / million tokens": 2, "Output price USD / million tokens": 6, "Median output tokens / second": 96.6450617140349, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:grok-4-7-high", "model": "Grok 4.7 (high)", "org": "SpaceXAI", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/grok-4-7-high", "metrics": { "index": 46.3321770885625 }, "details": { "Publisher model slug": "grok-4-7-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 500000, "Cost per index task (USD)": 2.726106691786027, "Input price USD / million tokens": 2, "Output price USD / million tokens": 6, "Median output tokens / second": 90.9124235268311, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mimo-v2-6-pro", "model": "MiMo-V2.6-Pro", "org": "Xiaomi", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/mimo-v2-6-pro", "metrics": { "index": 46.3242065310383 }, "details": { "Publisher model slug": "mimo-v2-6-pro", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.13322318937213493, "Input price USD / million tokens": 0.435, "Output price USD / million tokens": 0.87, "Median output tokens / second": 54.7461629430781, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-astra-low", "model": "GPT-6 Astra (low)", "org": "OpenAI", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-astra-low", "metrics": { "index": 45.7819243120341 }, "details": { "Publisher model slug": "gpt-6-astra-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.8175139285656057, "Input price USD / million tokens": 10, "Output price USD / million tokens": 50, "Median output tokens / second": 53.4224571605319, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-8-max", "model": "Qwen3.8 Max (0902)", "org": "Alibaba", "settings": "0902", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-8-max", "metrics": { "index": 45.4152084980521 }, "details": { "Publisher model slug": "qwen3-8-max", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 983616, "Cost per index task (USD)": 5.408509428374016, "Input price USD / million tokens": 2, "Output price USD / million tokens": 6, "Median output tokens / second": 41.8032457118835, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:deepseek-v4-1-flash-1017", "model": "DeepSeek V4.1 Flash 1017 (max)", "org": "DeepSeek", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/deepseek-v4-1-flash-1017", "metrics": { "index": 45.0965048681912 }, "details": { "Publisher model slug": "deepseek-v4-1-flash-1017", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.2620484273114172, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 1.2, "Median output tokens / second": 195.157086034074, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:glm-5-3", "model": "GLM-5.3 (max)", "org": "Z AI", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/glm-5-3", "metrics": { "index": 44.777392385614 }, "details": { "Publisher model slug": "glm-5-3", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 2.0056375150449584, "Input price USD / million tokens": 1.4, "Output price USD / million tokens": 4.4, "Median output tokens / second": 84.9535132213852, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ling-3-1-flash-1019", "model": "Ling 3.1 Flash 1019", "org": "InclusionAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ling-3-1-flash-1019", "metrics": { "index": 43.8973584978265 }, "details": { "Publisher model slug": "ling-3-1-flash-1019", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1048576, "Cost per index task (USD)": 1.4886425967752979, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 0.9, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:step-5", "model": "Step 5 Preview", "org": "StepFun", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/step-5", "metrics": { "index": 43.7343049141614 }, "details": { "Publisher model slug": "step-5", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 1.0264068637684054, "Input price USD / million tokens": 1, "Output price USD / million tokens": 2.7, "Median output tokens / second": 228.929004650371, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-haiku-5-5", "model": "Claude Haiku 5.5 (max)", "org": "Anthropic", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-haiku-5-5", "metrics": { "index": 43.3950199670746 }, "details": { "Publisher model slug": "claude-haiku-5-5", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.21276971216992022, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 235.628653400028, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:minimax-m3-1-flash", "model": "MiniMax-M3.1-Flash", "org": "MiniMax", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/minimax-m3-1-flash", "metrics": { "index": 42.7057475126235 }, "details": { "Publisher model slug": "minimax-m3-1-flash", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.2201135746720313, "Input price USD / million tokens": 0.15, "Output price USD / million tokens": 0.6, "Median output tokens / second": 95.5531649197315, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-opus-5-5-low", "model": "Claude Opus 5.5 (low with fallback)", "org": "Anthropic", "settings": "Low, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-opus-5-5-low", "metrics": { "index": 42.3077781466168 }, "details": { "Publisher model slug": "claude-opus-5-5-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.551180473909146, "Input price USD / million tokens": 4, "Output price USD / million tokens": 20, "Median output tokens / second": 81.2213306127604, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:grok-4-7-low", "model": "Grok 4.7 (low)", "org": "SpaceXAI", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/grok-4-7-low", "metrics": { "index": 42.2166560223448 }, "details": { "Publisher model slug": "grok-4-7-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 500000, "Cost per index task (USD)": 1.2479664056943975, "Input price USD / million tokens": 2, "Output price USD / million tokens": 6, "Median output tokens / second": 78.0407473032213, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-1-sol-low", "model": "GPT-6.1 Sol (low)", "org": "OpenAI", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-1-sol-low", "metrics": { "index": 42.0835618555848 }, "details": { "Publisher model slug": "gpt-6-1-sol-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.13075190869859377, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 47.7811712392998, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-6-terra", "model": "GPT-5.6 Terra (max)", "org": "OpenAI", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-5-6-terra", "metrics": { "index": 42.0829248239494 }, "details": { "Publisher model slug": "gpt-5-6-terra", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.3987235612691513, "Input price USD / million tokens": 2, "Output price USD / million tokens": 12, "Median output tokens / second": 125.632501685259, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:glm-5-3-flash", "model": "GLM-5.3-Flash", "org": "Z AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/glm-5-3-flash", "metrics": { "index": 41.807466113455 }, "details": { "Publisher model slug": "glm-5-3-flash", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.2835948232103717, "Input price USD / million tokens": 0.15, "Output price USD / million tokens": 0.5, "Median output tokens / second": 68.8605818769465, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-haiku-5-5-xhigh", "model": "Claude Haiku 5.5 (xhigh)", "org": "Anthropic", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-haiku-5-5-xhigh", "metrics": { "index": 41.2489304677024 }, "details": { "Publisher model slug": "claude-haiku-5-5-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.12378561092096102, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 190.199096022514, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ling-3-1-flash", "model": "Ling 3.1 Flash", "org": "InclusionAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ling-3-1-flash", "metrics": { "index": 41.0906200397391 }, "details": { "Publisher model slug": "ling-3-1-flash", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 1.010607821950605, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 0.9, "Median output tokens / second": 218.946635962805, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemini-3-8-flash", "model": "Gemini 3.8 Flash (high)", "org": "Google", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemini-3-8-flash", "metrics": { "index": 40.9262321765904 }, "details": { "Publisher model slug": "gemini-3-8-flash", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.2427947606950427, "Input price USD / million tokens": 0.75, "Output price USD / million tokens": 3.75, "Median output tokens / second": 141.403267108262, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-sonnet-5-5-medium", "model": "Claude Sonnet 5.5 (medium with fallback)", "org": "Anthropic", "settings": "Medium, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-sonnet-5-5-medium", "metrics": { "index": 40.838652287929 }, "details": { "Publisher model slug": "claude-sonnet-5-5-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.48326378260543096, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 103.948642622631, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-8-2-4t-a95b", "model": "Qwen3.8 2.4T A95B", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-8-2-4t-a95b", "metrics": { "index": 39.9461723105325 }, "details": { "Publisher model slug": "qwen3-8-2-4t-a95b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262000, "Cost per index task (USD)": 2.1551619442434675, "Input price USD / million tokens": 2, "Output price USD / million tokens": 6, "Median output tokens / second": 43.0064806562278, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-8-flash-next", "model": "Qwen3.8-Flash-Next", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-8-flash-next", "metrics": { "index": 39.8223021864741 }, "details": { "Publisher model slug": "qwen3-8-flash-next", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.3721760025302965, "Input price USD / million tokens": 0.15, "Output price USD / million tokens": 0.47, "Median output tokens / second": 51.513857139104, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemini-3-8-flash-medium", "model": "Gemini 3.8 Flash (medium)", "org": "Google", "settings": "Medium", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemini-3-8-flash-medium", "metrics": { "index": 39.7740187061926 }, "details": { "Publisher model slug": "gemini-3-8-flash-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.9310379374376306, "Input price USD / million tokens": 0.75, "Output price USD / million tokens": 3.75, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:muse-spark-open", "model": "Muse Spark Open (xhigh)", "org": "Meta", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/muse-spark-open", "metrics": { "index": 38.8666662997805 }, "details": { "Publisher model slug": "muse-spark-open", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1048576, "Cost per index task (USD)": 0.8441853713458678, "Input price USD / million tokens": 1.25, "Output price USD / million tokens": 4.25, "Median output tokens / second": 150.852574838856, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mistral-large-4", "model": "Mistral Large 4 Preview", "org": "Mistral", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/mistral-large-4", "metrics": { "index": 38.3770408917494 }, "details": { "Publisher model slug": "mistral-large-4", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 524288, "Cost per index task (USD)": 1.1344391582388638, "Input price USD / million tokens": 1.36, "Output price USD / million tokens": 4.18, "Median output tokens / second": 94.5168991835342, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-luna", "model": "GPT-6 Luna (max)", "org": "OpenAI", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-luna", "metrics": { "index": 38.1245186869738 }, "details": { "Publisher model slug": "gpt-6-luna", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.06782295669812205, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 137.030135328048, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-6-terra-xhigh", "model": "GPT-5.6 Terra (xhigh)", "org": "OpenAI", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-5-6-terra-xhigh", "metrics": { "index": 37.9513370742927 }, "details": { "Publisher model slug": "gpt-5-6-terra-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.6317911346924275, "Input price USD / million tokens": 2, "Output price USD / million tokens": 12, "Median output tokens / second": 111.848036749536, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mimo-v2-6-flash", "model": "MiMo-V2.6-Flash", "org": "Xiaomi", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/mimo-v2-6-flash", "metrics": { "index": 37.8843590141754 }, "details": { "Publisher model slug": "mimo-v2-6-flash", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.06237567475410205, "Input price USD / million tokens": 0.14, "Output price USD / million tokens": 0.28, "Median output tokens / second": 59.4249999270331, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-haiku-5-5-high", "model": "Claude Haiku 5.5 (high)", "org": "Anthropic", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-haiku-5-5-high", "metrics": { "index": 37.8240211671094 }, "details": { "Publisher model slug": "claude-haiku-5-5-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.07942671568737021, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 189.257505191959, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:deepseek-v4-pro", "model": "DeepSeek V4 Pro 0813 (max)", "org": "DeepSeek", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/deepseek-v4-pro", "metrics": { "index": 35.9967791278402 }, "details": { "Publisher model slug": "deepseek-v4-pro", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.6740219040435134, "Input price USD / million tokens": 1.32, "Output price USD / million tokens": 3.96, "Median output tokens / second": 97.0337681361021, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-sonnet-5-5-low", "model": "Claude Sonnet 5.5 (low with fallback)", "org": "Anthropic", "settings": "Low, Default Fallback", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-sonnet-5-5-low", "metrics": { "index": 35.8681594577587 }, "details": { "Publisher model slug": "claude-sonnet-5-5-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.3451610270666562, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": 103.954198754368, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:agnes-2-5-pro-beta", "model": "Agnes 2.5 Pro Beta", "org": "Sapiens AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/agnes-2-5-pro-beta", "metrics": { "index": 35.2413300369053 }, "details": { "Publisher model slug": "agnes-2-5-pro-beta", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.3, "Median output tokens / second": 113.084196380469, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:deepseek-v4-flash-vision", "model": "DeepSeek V4 Flash Vision (max)", "org": "DeepSeek", "settings": "Max", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/deepseek-v4-flash-vision", "metrics": { "index": 34.8390628035969 }, "details": { "Publisher model slug": "deepseek-v4-flash-vision", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.037910627432751, "Input price USD / million tokens": 0.44, "Output price USD / million tokens": 1.32, "Median output tokens / second": 189.843426217937, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-luna-xhigh", "model": "GPT-6 Luna (xhigh)", "org": "OpenAI", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-luna-xhigh", "metrics": { "index": 34.5587320217785 }, "details": { "Publisher model slug": "gpt-6-luna-xhigh", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.042183547923951084, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 126.122927009702, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-haiku-5-5-medium", "model": "Claude Haiku 5.5 (medium)", "org": "Anthropic", "settings": "Medium", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-haiku-5-5-medium", "metrics": { "index": 34.4646519474657 }, "details": { "Publisher model slug": "claude-haiku-5-5-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.04703084793955566, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 180.203198523279, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:glm-5-3-low", "model": "GLM-5.3 (low)", "org": "Z AI", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/glm-5-3-low", "metrics": { "index": 34.2990078159643 }, "details": { "Publisher model slug": "glm-5-3-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.8522174973386498, "Input price USD / million tokens": 1.4, "Output price USD / million tokens": 4.4, "Median output tokens / second": 89.1406606861554, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-6-terra-high", "model": "GPT-5.6 Terra (high)", "org": "OpenAI", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-5-6-terra-high", "metrics": { "index": 34.2376558067555 }, "details": { "Publisher model slug": "gpt-5-6-terra-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.3378727947949101, "Input price USD / million tokens": 2, "Output price USD / million tokens": 12, "Median output tokens / second": 115.650379504255, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:jt-4-1-flash-236b-a21b", "model": "JT-4.1 Flash 236B A21B", "org": "China Mobile", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/jt-4-1-flash-236b-a21b", "metrics": { "index": 33.9172581709394 }, "details": { "Publisher model slug": "jt-4-1-flash-236b-a21b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-8-27b", "model": "Qwen3.8 27B (xhigh)", "org": "Alibaba", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-8-27b", "metrics": { "index": 33.6762858057476 }, "details": { "Publisher model slug": "qwen3-8-27b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 1.0074896941565943, "Input price USD / million tokens": 0.5, "Output price USD / million tokens": 3, "Median output tokens / second": 43.538366446435, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:motif-3", "model": "Motif 3", "org": "Motif Technologies", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/motif-3", "metrics": { "index": 33.566562209723 }, "details": { "Publisher model slug": "motif-3", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemini-3-8-flash-low", "model": "Gemini 3.8 Flash (low)", "org": "Google", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemini-3-8-flash-low", "metrics": { "index": 33.4548106699132 }, "details": { "Publisher model slug": "gemini-3-8-flash-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.75, "Output price USD / million tokens": 3.75, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-luna-high", "model": "GPT-6 Luna (high)", "org": "OpenAI", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-luna-high", "metrics": { "index": 32.9282054150837 }, "details": { "Publisher model slug": "gpt-6-luna-high", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.02903157640372802, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 123.035249875314, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-3-codex", "model": "GPT-5.3 Codex (xhigh)", "org": "OpenAI", "settings": "Xhigh", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gpt-5-3-codex", "metrics": { "index": 32.5028174368983 }, "details": { "Publisher model slug": "gpt-5-3-codex", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 400000, "Cost per index task (USD)": null, "Input price USD / million tokens": 1.75, "Output price USD / million tokens": 14, "Median output tokens / second": 86.614023432458, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-horizon-375b-a23b", "model": "K2 Horizon 375B A23B", "org": "Institute of Foundation Models", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/k2-horizon-375b-a23b", "metrics": { "index": 30.5015302696723 }, "details": { "Publisher model slug": "k2-horizon-375b-a23b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 524288, "Cost per index task (USD)": 0, "Input price USD / million tokens": 0, "Output price USD / million tokens": 0, "Median output tokens / second": 909.189384541859, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:reflection-beam", "model": "Beam 1.0", "org": "Reflection", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/reflection-beam", "metrics": { "index": 30.3334426348039 }, "details": { "Publisher model slug": "reflection-beam", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1048576, "Cost per index task (USD)": 0.22624169286047544, "Input price USD / million tokens": 0.4, "Output price USD / million tokens": 1.2, "Median output tokens / second": 336.739579348529, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-6-terra-medium", "model": "GPT-5.6 Terra (medium)", "org": "OpenAI", "settings": "Medium", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-5-6-terra-medium", "metrics": { "index": 30.0938023419377 }, "details": { "Publisher model slug": "gpt-5-6-terra-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.18337295992969213, "Input price USD / million tokens": 2, "Output price USD / million tokens": 12, "Median output tokens / second": 109.129372900772, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-luna-medium", "model": "GPT-6 Luna (medium)", "org": "OpenAI", "settings": "Medium", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-luna-medium", "metrics": { "index": 29.9251773515762 }, "details": { "Publisher model slug": "gpt-6-luna-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.017476906067835942, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 91.5429083178543, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemini-3-1-pro-preview", "model": "Gemini 3.1 Pro Preview", "org": "Google", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemini-3-1-pro-preview", "metrics": { "index": 29.7185656132261 }, "details": { "Publisher model slug": "gemini-3-1-pro-preview", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 1.2985855086120641, "Input price USD / million tokens": 2, "Output price USD / million tokens": 12, "Median output tokens / second": 132.086124569807, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:claude-haiku-5-5-low", "model": "Claude Haiku 5.5 (low)", "org": "Anthropic", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/claude-haiku-5-5-low", "metrics": { "index": 29.4523685891139 }, "details": { "Publisher model slug": "claude-haiku-5-5-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.02445936101890973, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 176.329219792214, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:minimax-m3", "model": "MiniMax-M3", "org": "MiniMax", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/minimax-m3", "metrics": { "index": 29.2202932236316 }, "details": { "Publisher model slug": "minimax-m3", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.5076126905781733, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 1.2, "Median output tokens / second": 88.7245235883505, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nex-n2-pro", "model": "Nex-N2-Pro", "org": "Nex AGI", "settings": "Based on Qwen3.5-397B-A17B", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nex-n2-pro", "metrics": { "index": 28.1990972887912 }, "details": { "Publisher model slug": "nex-n2-pro", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-8-27b-medium", "model": "Qwen3.8 27B (medium)", "org": "Alibaba", "settings": "Medium", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-8-27b-medium", "metrics": { "index": 27.5508228231586 }, "details": { "Publisher model slug": "qwen3-8-27b-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 1.133063774030521, "Input price USD / million tokens": 0.5, "Output price USD / million tokens": 3, "Median output tokens / second": 49.7690953164986, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-6-terra-low", "model": "GPT-5.6 Terra (low)", "org": "OpenAI", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-5-6-terra-low", "metrics": { "index": 27.496203401716 }, "details": { "Publisher model slug": "gpt-5-6-terra-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.1444617614290821, "Input price USD / million tokens": 2, "Output price USD / million tokens": 12, "Median output tokens / second": 114.439091197719, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:jt-4-1-flash-236b-a21b-non-reasoning", "model": "JT-4.1 Flash 236B A21B (non-reasoning)", "org": "China Mobile", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/jt-4-1-flash-236b-a21b-non-reasoning", "metrics": { "index": 27.2926268381758 }, "details": { "Publisher model slug": "jt-4-1-flash-236b-a21b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:quasar-438b", "model": "Quasar 438B (max)", "org": "Multiverse Computing", "settings": "Max, Based on GLM-5.2", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/quasar-438b", "metrics": { "index": 26.7372536607816 }, "details": { "Publisher model slug": "quasar-438b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.11, "Output price USD / million tokens": 0.33, "Median output tokens / second": 171.587479318445, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:solar-pro4", "model": "Solar Pro 4", "org": "Upstage", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/solar-pro4", "metrics": { "index": 26.5600314812231 }, "details": { "Publisher model slug": "solar-pro4", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 512000, "Cost per index task (USD)": 0.7113065897513132, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 1.2, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nemotron-3-5-super-vl", "model": "Nemotron 3.5 Super VL", "org": "NVIDIA", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/nemotron-3-5-super-vl", "metrics": { "index": 26.4910658661689 }, "details": { "Publisher model slug": "nemotron-3-5-super-vl", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.18823665957318256, "Input price USD / million tokens": 0.225, "Output price USD / million tokens": 0.575, "Median output tokens / second": 453.383218759152, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:apodex-1-1", "model": "Apodex 1.1", "org": "Apodex", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/apodex-1-1", "metrics": { "index": 26.4103686249117 }, "details": { "Publisher model slug": "apodex-1-1", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": 0.4637245706928438, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 3, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-8-27b-low", "model": "Qwen3.8 27B (low)", "org": "Alibaba", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-8-27b-low", "metrics": { "index": 26.2047978189896 }, "details": { "Publisher model slug": "qwen3-8-27b-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 1.0482225365160391, "Input price USD / million tokens": 0.5, "Output price USD / million tokens": 3, "Median output tokens / second": 51.2494548038687, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-5-instant-06-26", "model": "GPT-5.5 Instant (June 2026)", "org": "OpenAI", "settings": "June 2026", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-5-5-instant-06-26", "metrics": { "index": 26.0135173401368 }, "details": { "Publisher model slug": "gpt-5-5-instant-06-26", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 400000, "Cost per index task (USD)": 0.6914853678439488, "Input price USD / million tokens": 5, "Output price USD / million tokens": 30, "Median output tokens / second": 154.739290629232, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:kimi-k2-7-code", "model": "Kimi K2.7 Code", "org": "Kimi", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/kimi-k2-7-code", "metrics": { "index": 25.8121062401836 }, "details": { "Publisher model slug": "kimi-k2-7-code", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.541190247118529, "Input price USD / million tokens": 0.95, "Output price USD / million tokens": 4, "Median output tokens / second": 59.9561235789781, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:inkling-small", "model": "Inkling Small", "org": "Thinking Machines", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/inkling-small", "metrics": { "index": 25.6582584411942 }, "details": { "Publisher model slug": "inkling-small", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.0894649973284348, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 1.2, "Median output tokens / second": 223.99836135598, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-horizon-mova-36b-a4b", "model": "K2 Horizon MoVA 36B A4B", "org": "Institute of Foundation Models", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/k2-horizon-mova-36b-a4b", "metrics": { "index": 25.3037790731435 }, "details": { "Publisher model slug": "k2-horizon-mova-36b-a4b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 524288, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:hy3", "model": "Hy3", "org": "Tencent", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/hy3", "metrics": { "index": 25.2972775731571 }, "details": { "Publisher model slug": "hy3", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.07179763081859954, "Input price USD / million tokens": 0.136, "Output price USD / million tokens": 0.5549999999999999, "Median output tokens / second": 82.3949792753023, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-7-plus", "model": "Qwen3.7 Plus", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-7-plus", "metrics": { "index": 25.1622215821984 }, "details": { "Publisher model slug": "qwen3-7-plus", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.21702592259078568, "Input price USD / million tokens": 0.4, "Output price USD / million tokens": 1.6, "Median output tokens / second": 59.2185154630422, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:inkling", "model": "Inkling (xhigh)", "org": "Thinking Machines", "settings": "Xhigh", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/inkling", "metrics": { "index": 24.9847810999384 }, "details": { "Publisher model slug": "inkling", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 1, "Output price USD / million tokens": 4.05, "Median output tokens / second": 194.414009820784, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:solar-open2-250b", "model": "Solar Open2 250B", "org": "Upstage", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/solar-open2-250b", "metrics": { "index": 24.7365297627523 }, "details": { "Publisher model slug": "solar-open2-250b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1048576, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ling-3-0-flash-vl", "model": "Ling-3.0-flash-VL", "org": "InclusionAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ling-3-0-flash-vl", "metrics": { "index": 24.5693054077875 }, "details": { "Publisher model slug": "ling-3-0-flash-vl", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:solar-mini4", "model": "Solar Mini 4", "org": "Upstage", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/solar-mini4", "metrics": { "index": 24.0522018844171 }, "details": { "Publisher model slug": "solar-mini4", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1048576, "Cost per index task (USD)": 0.3655852120863359, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.4, "Median output tokens / second": 95.2246437255575, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-3-ultra-550b-a55b", "model": "Nemotron 3 Ultra", "org": "NVIDIA", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-ultra-550b-a55b", "metrics": { "index": 22.9279123422858 }, "details": { "Publisher model slug": "nvidia-nemotron-3-ultra-550b-a55b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": 0.4696973787770703, "Input price USD / million tokens": 0.6, "Output price USD / million tokens": 2.7, "Median output tokens / second": 154.20990181002, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ling-3-0-flash-fin", "model": "Ling-3.0-flash-Fin", "org": "InclusionAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ling-3-0-flash-fin", "metrics": { "index": 22.5965828940594 }, "details": { "Publisher model slug": "ling-3-0-flash-fin", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemini-3-5-flash-lite", "model": "Gemini 3.5 Flash-Lite", "org": "Google", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemini-3-5-flash-lite", "metrics": { "index": 22.1685424839812 }, "details": { "Publisher model slug": "gemini-3-5-flash-lite", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.1927430131409478, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 2.5, "Median output tokens / second": 430.164873198974, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:g9v3-39a5b", "model": "G9v3-39A5B", "org": "AI9Stars", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/g9v3-39a5b", "metrics": { "index": 21.8316172646825 }, "details": { "Publisher model slug": "g9v3-39a5b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:kat-coder-pro-v2", "model": "KAT-Coder-Pro V2", "org": "KwaiKAT", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/kat-coder-pro-v2", "metrics": { "index": 21.7045082420985 }, "details": { "Publisher model slug": "kat-coder-pro-v2", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-luna-low", "model": "GPT-6 Luna (low)", "org": "OpenAI", "settings": "Low", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-luna-low", "metrics": { "index": 21.5261980080866 }, "details": { "Publisher model slug": "gpt-6-luna-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.004516607569410252, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 111.276631757707, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:a-x-k2", "model": "A.X-K2", "org": "SK Telecom", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/a-x-k2", "metrics": { "index": 21.4612645385074 }, "details": { "Publisher model slug": "a-x-k2", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-397b-a17b-non-reasoning", "model": "Qwen3.5 397B A17B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-397b-a17b-non-reasoning", "metrics": { "index": 21.4366547759392 }, "details": { "Publisher model slug": "qwen3-5-397b-a17b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.6, "Output price USD / million tokens": 3.6, "Median output tokens / second": 59.1481612242436, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nemotron-3-5-bolt-74b-a9b", "model": "Nemotron 3.5 Bolt 74B-A9B", "org": "NVIDIA", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/nemotron-3-5-bolt-74b-a9b", "metrics": { "index": 21.4072054842003 }, "details": { "Publisher model slug": "nemotron-3-5-bolt-74b-a9b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-6-terra-non-reasoning", "model": "GPT-5.6 Terra (non-reasoning)", "org": "OpenAI", "settings": "Non-reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-5-6-terra-non-reasoning", "metrics": { "index": 20.7797013584503 }, "details": { "Publisher model slug": "gpt-5-6-terra-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.13972397093498629, "Input price USD / million tokens": 2, "Output price USD / million tokens": 12, "Median output tokens / second": 114.783259288906, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-horizon-7b", "model": "K2 Horizon 7B", "org": "Institute of Foundation Models", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/k2-horizon-7b", "metrics": { "index": 20.5958819421354 }, "details": { "Publisher model slug": "k2-horizon-7b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 524288, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-omni-plus", "model": "Qwen3.5 Omni Plus", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-omni-plus", "metrics": { "index": 20.3839890713777 }, "details": { "Publisher model slug": "qwen3-5-omni-plus", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.4, "Output price USD / million tokens": 4.8, "Median output tokens / second": 112.347673393648, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:deepseek-v4-pro-non-reasoning", "model": "DeepSeek V4 Pro 0813 (non-reasoning)", "org": "DeepSeek", "settings": "Non-reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/deepseek-v4-pro-non-reasoning", "metrics": { "index": 20.3605101726847 }, "details": { "Publisher model slug": "deepseek-v4-pro-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.35547065335335964, "Input price USD / million tokens": 1.32, "Output price USD / million tokens": 3.96, "Median output tokens / second": 96.3641918755705, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:o3", "model": "o3", "org": "OpenAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/o3", "metrics": { "index": 20.200754789201 }, "details": { "Publisher model slug": "o3", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 200000, "Cost per index task (USD)": null, "Input price USD / million tokens": 2, "Output price USD / million tokens": 8, "Median output tokens / second": 133.98728354155, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-8-27b-non-reasoning", "model": "Qwen3.8 27B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-8-27b-non-reasoning", "metrics": { "index": 20.1502427055595 }, "details": { "Publisher model slug": "qwen3-8-27b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 2.487757391889078, "Input price USD / million tokens": 0.5, "Output price USD / million tokens": 3, "Median output tokens / second": 47.7426804908045, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k-exaone-2-0-0803", "model": "K-EXAONE 2.0", "org": "LG AI Research", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/k-exaone-2-0-0803", "metrics": { "index": 19.6904943371003 }, "details": { "Publisher model slug": "k-exaone-2-0-0803", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:longcat-2-0", "model": "LongCat 2.0", "org": "LongCat", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/longcat-2-0", "metrics": { "index": 19.1123256610623 }, "details": { "Publisher model slug": "longcat-2-0", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.14624850952440743, "Input price USD / million tokens": 0.75, "Output price USD / million tokens": 2.95, "Median output tokens / second": 76.7873682312517, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:jt-35b-flash", "model": "JT-35B-Flash", "org": "China Mobile", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/jt-35b-flash", "metrics": { "index": 18.6626376411117 }, "details": { "Publisher model slug": "jt-35b-flash", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-luna-non-reasoning", "model": "GPT-6 Luna (non-reasoning)", "org": "OpenAI", "settings": "Non-reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-6-luna-non-reasoning", "metrics": { "index": 18.4678465935701 }, "details": { "Publisher model slug": "gpt-6-luna-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": 0.011345057120527267, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.5, "Median output tokens / second": 111.184932124848, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-397b-a17b", "model": "Qwen3.5 397B A17B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-5-397b-a17b", "metrics": { "index": 18.4204249294815 }, "details": { "Publisher model slug": "qwen3-5-397b-a17b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": 0.47457695829894075, "Input price USD / million tokens": 0.6, "Output price USD / million tokens": 3.6, "Median output tokens / second": 58.3084062588423, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-6-35b-a3b", "model": "Qwen3.6 35B A3B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-6-35b-a3b", "metrics": { "index": 18.2289900189843 }, "details": { "Publisher model slug": "qwen3-6-35b-a3b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": 0.4750534562511306, "Input price USD / million tokens": 0.375, "Output price USD / million tokens": 2.25, "Median output tokens / second": 136.420106486181, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-122b-a10b-non-reasoning", "model": "Qwen3.5 122B A10B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-122b-a10b-non-reasoning", "metrics": { "index": 17.7480154665397 }, "details": { "Publisher model slug": "qwen3-5-122b-a10b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.4, "Output price USD / million tokens": 3.2, "Median output tokens / second": 142.452776677266, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:muse-glimmer", "model": "Muse Glimmer (high)", "org": "Meta", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/muse-glimmer", "metrics": { "index": 17.4754176089465 }, "details": { "Publisher model slug": "muse-glimmer", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": 0.05755125039078422, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 1.2, "Median output tokens / second": 118.909878525429, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:doubao-seed-code", "model": "Doubao Seed Code", "org": "ByteDance Seed", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/doubao-seed-code", "metrics": { "index": 16.9467715940932 }, "details": { "Publisher model slug": "doubao-seed-code", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-26b-a4b", "model": "Gemma 4 26B A4B", "org": "Google", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-26b-a4b", "metrics": { "index": 16.671930040086 }, "details": { "Publisher model slug": "gemma-4-26b-a4b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.34, "Median output tokens / second": 81.4730889753222, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ring-2-6-1t", "model": "Ring-2.6-1T", "org": "InclusionAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ring-2-6-1t", "metrics": { "index": 16.6172363768618 }, "details": { "Publisher model slug": "ring-2-6-1t", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": 0.2871573189066596, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 2.5, "Median output tokens / second": 120.685133997056, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-horizon-3-7b", "model": "K2 Horizon 3.7B", "org": "Institute of Foundation Models", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/k2-horizon-3-7b", "metrics": { "index": 15.6103951330976 }, "details": { "Publisher model slug": "k2-horizon-3-7b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 524288, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-122b-a10b", "model": "Qwen3.5 122B A10B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-5-122b-a10b", "metrics": { "index": 15.5694009312969 }, "details": { "Publisher model slug": "qwen3-5-122b-a10b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": 0.3247468637357143, "Input price USD / million tokens": 0.4, "Output price USD / million tokens": 3.2, "Median output tokens / second": 130.639708775809, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-6-35b-a3b-non-reasoning", "model": "Qwen3.6 35B A3B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-6-35b-a3b-non-reasoning", "metrics": { "index": 15.2320772647744 }, "details": { "Publisher model slug": "qwen3-6-35b-a3b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.375, "Output price USD / million tokens": 2.25, "Median output tokens / second": 156.01775752522, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-31b", "model": "Gemma 4 31B", "org": "Google", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemma-4-31b", "metrics": { "index": 14.6689500745828 }, "details": { "Publisher model slug": "gemma-4-31b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0, "Input price USD / million tokens": 0, "Output price USD / million tokens": 0, "Median output tokens / second": 35.9477059936716, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ernie-5-0-thinking-preview", "model": "ERNIE 5.0 Thinking Preview", "org": "Baidu", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/ernie-5-0-thinking-preview", "metrics": { "index": 14.2551784975475 }, "details": { "Publisher model slug": "ernie-5-0-thinking-preview", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mistral-medium-3-5", "model": "Mistral Medium 3.5", "org": "Mistral", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/mistral-medium-3-5", "metrics": { "index": 14.1889227802691 }, "details": { "Publisher model slug": "mistral-medium-3-5", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.5020382394983653, "Input price USD / million tokens": 1.5, "Output price USD / million tokens": 7.5, "Median output tokens / second": 159.137333513867, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-12b", "model": "Gemma 4 12B", "org": "Google", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-12b", "metrics": { "index": 14.1847516647736 }, "details": { "Publisher model slug": "gemma-4-12b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.3, "Median output tokens / second": 119.763705410717, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-pro-reasoning-medium", "model": "Nova 2.0 Pro Preview (medium)", "org": "Amazon", "settings": "Medium", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-pro-reasoning-medium", "metrics": { "index": 14.1554948903666 }, "details": { "Publisher model slug": "nova-2-0-pro-reasoning-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 1.25, "Output price USD / million tokens": 10, "Median output tokens / second": 124.880620946756, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-31b-non-reasoning", "model": "Gemma 4 31B (non-reasoning)", "org": "Google", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-31b-non-reasoning", "metrics": { "index": 13.9198819005287 }, "details": { "Publisher model slug": "gemma-4-31b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.16499999999999998, "Output price USD / million tokens": 0.4, "Median output tokens / second": 51.5093447189457, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-omni-reasoning-medium", "model": "Nova 2.0 Omni (medium)", "org": "Amazon", "settings": "Medium", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-omni-reasoning-medium", "metrics": { "index": 13.6465213725579 }, "details": { "Publisher model slug": "nova-2-0-omni-reasoning-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:apriel-v1-6-15b-thinker", "model": "Apriel-v1.6-15B-Thinker", "org": "ServiceNow", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/apriel-v1-6-15b-thinker", "metrics": { "index": 13.401396407375 }, "details": { "Publisher model slug": "apriel-v1-6-15b-thinker", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-lite-reasoning", "model": "Nova 2.0 Lite (high)", "org": "Amazon", "settings": "High", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-lite-reasoning", "metrics": { "index": 13.3740793005281 }, "details": { "Publisher model slug": "nova-2-0-lite-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 2.5, "Median output tokens / second": 182.150044726712, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-9b-non-reasoning", "model": "Qwen3.5 9B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-9b-non-reasoning", "metrics": { "index": 13.2659955355465 }, "details": { "Publisher model slug": "qwen3-5-9b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.17, "Output price USD / million tokens": 0.25, "Median output tokens / second": 78.2421271088243, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:exaone-4-5-33b", "model": "EXAONE 4.5 33B", "org": "LG AI Research", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/exaone-4-5-33b", "metrics": { "index": 13.2058394009382 }, "details": { "Publisher model slug": "exaone-4-5-33b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:command-a-plus", "model": "Command A+", "org": "Cohere", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/command-a-plus", "metrics": { "index": 13.1335002685349 }, "details": { "Publisher model slug": "command-a-plus", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 192000, "Cost per index task (USD)": 0, "Input price USD / million tokens": 0, "Output price USD / million tokens": 0, "Median output tokens / second": 148.76030294507, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-26b-a4b-non-reasoning", "model": "Gemma 4 26B A4B (non-reasoning)", "org": "Google", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-26b-a4b-non-reasoning", "metrics": { "index": 13.1303772801787 }, "details": { "Publisher model slug": "gemma-4-26b-a4b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.115, "Output price USD / million tokens": 0.4, "Median output tokens / second": 95.7153029654662, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-4b", "model": "Qwen3.5 4B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-4b", "metrics": { "index": 13.1248174602542 }, "details": { "Publisher model slug": "qwen3-5-4b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.03, "Output price USD / million tokens": 0.15, "Median output tokens / second": 16.8165162279045, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nemotron-3-5-lightning", "model": "Nemotron 3.5 Lightning", "org": "NVIDIA", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/nemotron-3-5-lightning", "metrics": { "index": 12.8572306217144 }, "details": { "Publisher model slug": "nemotron-3-5-lightning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.07948959749314857, "Input price USD / million tokens": 0.06, "Output price USD / million tokens": 0.2, "Median output tokens / second": 287.766283853996, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-pro-reasoning-low", "model": "Nova 2.0 Pro Preview (low)", "org": "Amazon", "settings": "Low", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-pro-reasoning-low", "metrics": { "index": 12.7754507441167 }, "details": { "Publisher model slug": "nova-2-0-pro-reasoning-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 1.25, "Output price USD / million tokens": 10, "Median output tokens / second": 119.286729779229, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-2-30b", "model": "Granite 4.2 30B", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/granite-4-2-30b", "metrics": { "index": 12.68315341158 }, "details": { "Publisher model slug": "granite-4-2-30b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": 0.0716641484962871, "Input price USD / million tokens": 0.16, "Output price USD / million tokens": 0.65, "Median output tokens / second": 74.5264545347446, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-lite-reasoning-medium", "model": "Nova 2.0 Lite (medium)", "org": "Amazon", "settings": "Medium", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-lite-reasoning-medium", "metrics": { "index": 12.4823794919822 }, "details": { "Publisher model slug": "nova-2-0-lite-reasoning-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 2.5, "Median output tokens / second": 176.690110811972, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-omni-flash", "model": "Qwen3.5 Omni Flash", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-omni-flash", "metrics": { "index": 12.4740069826994 }, "details": { "Publisher model slug": "qwen3-5-omni-flash", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.8, "Median output tokens / second": 233.971040448268, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:minicpm5-2b", "model": "MiniCPM5-2B", "org": "OpenBMB", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/minicpm5-2b", "metrics": { "index": 12.463354831996 }, "details": { "Publisher model slug": "minicpm5-2b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mercury-2-5", "model": "Mercury 2.5", "org": "Inception", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/mercury-2-5", "metrics": { "index": 12.3408453757638 }, "details": { "Publisher model slug": "mercury-2-5", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 260000, "Cost per index task (USD)": 0.11668472036336801, "Input price USD / million tokens": 0.25, "Output price USD / million tokens": 0.75, "Median output tokens / second": 689.382795588267, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:jt-mini", "model": "JT-MINI", "org": "China Mobile", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/jt-mini", "metrics": { "index": 12.2117782837313 }, "details": { "Publisher model slug": "jt-mini", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:magistral-medium-2509", "model": "Magistral Medium 1.2", "org": "Mistral", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/magistral-medium-2509", "metrics": { "index": 11.8254509953707 }, "details": { "Publisher model slug": "magistral-medium-2509", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-lite-reasoning-low", "model": "Nova 2.0 Lite (low)", "org": "Amazon", "settings": "Low", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-lite-reasoning-low", "metrics": { "index": 11.8041280807831 }, "details": { "Publisher model slug": "nova-2-0-lite-reasoning-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 2.5, "Median output tokens / second": 175.02399680044, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:hypernova-60b", "model": "HyperNova 60B 2605 (high)", "org": "Multiverse Computing", "settings": "High, Based on gpt-oss-120b", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/hypernova-60b", "metrics": { "index": 11.7380211033452 }, "details": { "Publisher model slug": "hypernova-60b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nemotron-cascade-2-30b-a3b", "model": "Nemotron Cascade 2 30B A3B", "org": "NVIDIA", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nemotron-cascade-2-30b-a3b", "metrics": { "index": 11.6708359406601 }, "details": { "Publisher model slug": "nemotron-cascade-2-30b-a3b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-oss-120b", "model": "gpt-oss-120b (high)", "org": "OpenAI", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-oss-120b", "metrics": { "index": 11.6028431512592 }, "details": { "Publisher model slug": "gpt-oss-120b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": 0.10755929276182966, "Input price USD / million tokens": 0.15, "Output price USD / million tokens": 0.6, "Median output tokens / second": 208.118818775637, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-think-v2", "model": "K2 Think V2", "org": "Institute of Foundation Models", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/k2-think-v2", "metrics": { "index": 11.4956908518183 }, "details": { "Publisher model slug": "k2-think-v2", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:longcat-flash-lite", "model": "LongCat Flash Lite", "org": "LongCat", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/longcat-flash-lite", "metrics": { "index": 11.4709079965186 }, "details": { "Publisher model slug": "longcat-flash-lite", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:hyperclova-x-seed-think-32b", "model": "HyperCLOVA X SEED Think (32B)", "org": "Naver", "settings": "32B", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/hyperclova-x-seed-think-32b", "metrics": { "index": 11.3648115832519 }, "details": { "Publisher model slug": "hyperclova-x-seed-think-32b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mistral-small-4", "model": "Mistral Small 4", "org": "Mistral", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/mistral-small-4", "metrics": { "index": 11.2675794141469 }, "details": { "Publisher model slug": "mistral-small-4", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.01504679036588765, "Input price USD / million tokens": 0.15, "Output price USD / million tokens": 0.6, "Median output tokens / second": 183.133592658442, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-next-80b-a3b-reasoning", "model": "Qwen3 Next 80B A3B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-next-80b-a3b-reasoning", "metrics": { "index": 11.1973684395653 }, "details": { "Publisher model slug": "qwen3-next-80b-a3b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-9b", "model": "Qwen3.5 9B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-5-9b", "metrics": { "index": 11.1961650827156 }, "details": { "Publisher model slug": "qwen3-5-9b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": 0.21258346272934256, "Input price USD / million tokens": 0.135, "Output price USD / million tokens": 0.2, "Median output tokens / second": 58.8294578625926, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-omni-reasoning-low", "model": "Nova 2.0 Omni (low)", "org": "Amazon", "settings": "Low", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-omni-reasoning-low", "metrics": { "index": 11.1140626232787 }, "details": { "Publisher model slug": "nova-2-0-omni-reasoning-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-2-8b", "model": "Granite 4.2 8B", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/granite-4-2-8b", "metrics": { "index": 11.0812712267815 }, "details": { "Publisher model slug": "granite-4-2-8b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": 0.023596091491929188, "Input price USD / million tokens": 0.06, "Output price USD / million tokens": 0.25, "Median output tokens / second": 86.6189047519768, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ling-3-0-tiny", "model": "Ling 3.0 Tiny", "org": "InclusionAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ling-3-0-tiny", "metrics": { "index": 11.0636233769305 }, "details": { "Publisher model slug": "ling-3-0-tiny", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": 0, "Input price USD / million tokens": 0, "Output price USD / million tokens": 0, "Median output tokens / second": 57.280498625112, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mi-dm-k-2-5-pro-dec28", "model": "Mi:dm K 2.5 Pro", "org": "Korea Telecom", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/mi-dm-k-2-5-pro-dec28", "metrics": { "index": 11.0361387091137 }, "details": { "Publisher model slug": "mi-dm-k-2-5-pro-dec28", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:g9v3-3b", "model": "G9v3-3B", "org": "AI9Stars", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/g9v3-3b", "metrics": { "index": 10.841600440564 }, "details": { "Publisher model slug": "g9v3-3b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:trinity-large-thinking", "model": "Trinity Large Thinking", "org": "Arcee AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/trinity-large-thinking", "metrics": { "index": 10.8242425924218 }, "details": { "Publisher model slug": "trinity-large-thinking", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 512000, "Cost per index task (USD)": 0.11883788334567012, "Input price USD / million tokens": 0.25, "Output price USD / million tokens": 0.8, "Median output tokens / second": 356.932301440532, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-4b-non-reasoning", "model": "Qwen3.5 4B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-4b-non-reasoning", "metrics": { "index": 10.812307683746 }, "details": { "Publisher model slug": "qwen3-5-4b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.03, "Output price USD / million tokens": 0.15, "Median output tokens / second": 11.121238848616, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:intellect-3", "model": "INTELLECT-3", "org": "Prime Intellect", "settings": "Based on GLM-4.5-Air", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/intellect-3", "metrics": { "index": 10.6061102858178 }, "details": { "Publisher model slug": "intellect-3", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131100, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:solar-open-100b-reasoning", "model": "Solar Open 100B", "org": "Upstage", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/solar-open-100b-reasoning", "metrics": { "index": 10.3644352600556 }, "details": { "Publisher model slug": "solar-open-100b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nemotron-3-nano-omni-30b-a3b", "model": "Nemotron 3 Nano Omni 30B A3B", "org": "NVIDIA", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nemotron-3-nano-omni-30b-a3b", "metrics": { "index": 10.2506761710987 }, "details": { "Publisher model slug": "nemotron-3-nano-omni-30b-a3b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.4, "Output price USD / million tokens": 1.365, "Median output tokens / second": 233.880941223035, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-oss-120b-low", "model": "gpt-oss-120b (low)", "org": "OpenAI", "settings": "Low", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gpt-oss-120b-low", "metrics": { "index": 10.2105650347196 }, "details": { "Publisher model slug": "gpt-oss-120b-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.125, "Output price USD / million tokens": 0.595, "Median output tokens / second": 262.662693125677, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-pro", "model": "Nova 2.0 Pro Preview (non-reasoning)", "org": "Amazon", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-pro", "metrics": { "index": 9.95904965562327 }, "details": { "Publisher model slug": "nova-2-0-pro", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 1.25, "Output price USD / million tokens": 10, "Median output tokens / second": 110.929377214353, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-oss-20b-low", "model": "gpt-oss-20b (low)", "org": "OpenAI", "settings": "Low", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gpt-oss-20b-low", "metrics": { "index": 9.95295342568922 }, "details": { "Publisher model slug": "gpt-oss-20b-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.07, "Output price USD / million tokens": 0.3, "Median output tokens / second": 175.254947323444, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:north-mini-code", "model": "North Mini Code", "org": "Cohere", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/north-mini-code", "metrics": { "index": 9.91462776590372 }, "details": { "Publisher model slug": "north-mini-code", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0, "Input price USD / million tokens": 0, "Output price USD / million tokens": 0, "Median output tokens / second": 60.2585541194219, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-v2", "model": "K2-V2 (high)", "org": "Institute of Foundation Models", "settings": "High", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/k2-v2", "metrics": { "index": 9.87390600007694 }, "details": { "Publisher model slug": "k2-v2", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 512000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-next-80b-a3b-instruct", "model": "Qwen3 Next 80B A3B", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-next-80b-a3b-instruct", "metrics": { "index": 9.64335007559183 }, "details": { "Publisher model slug": "qwen3-next-80b-a3b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 1.1, "Median output tokens / second": 96.2213223760374, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:diffusiongemma-26b-a4b", "model": "DiffusionGemma 26B A4B", "org": "Google", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/diffusiongemma-26b-a4b", "metrics": { "index": 9.51014305662543 }, "details": { "Publisher model slug": "diffusiongemma-26b-a4b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-12b-non-reasoning", "model": "Gemma 4 12B (non-reasoning)", "org": "Google", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-12b-non-reasoning", "metrics": { "index": 9.37989472421056 }, "details": { "Publisher model slug": "gemma-4-12b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.3, "Median output tokens / second": 127.441316381844, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:llama-4-maverick", "model": "Llama 4 Maverick", "org": "Meta", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/llama-4-maverick", "metrics": { "index": 9.29902687437199 }, "details": { "Publisher model slug": "llama-4-maverick", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.038299797707028495, "Input price USD / million tokens": 0.25, "Output price USD / million tokens": 0.9199999999999999, "Median output tokens / second": 52.1174143377827, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-coder-next", "model": "Qwen3 Coder Next", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/qwen3-coder-next", "metrics": { "index": 9.23682203219174 }, "details": { "Publisher model slug": "qwen3-coder-next", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.46969565354948617, "Input price USD / million tokens": 0.31, "Output price USD / million tokens": 1, "Median output tokens / second": 87.6171538949575, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:motif-2-12-7b", "model": "Motif-2-12.7B", "org": "Motif Technologies", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/motif-2-12-7b", "metrics": { "index": 9.18748556817538 }, "details": { "Publisher model slug": "motif-2-12-7b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-premier", "model": "Nova Premier", "org": "Amazon", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-premier", "metrics": { "index": 9.15908119897819 }, "details": { "Publisher model slug": "nova-premier", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-2-3b", "model": "Granite 4.2 3B", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/granite-4-2-3b", "metrics": { "index": 9.05588232376756 }, "details": { "Publisher model slug": "granite-4-2-3b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": 0.005987243224456071, "Input price USD / million tokens": 0.03, "Output price USD / million tokens": 0.12, "Median output tokens / second": 99.4201214723698, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-v2-medium", "model": "K2-V2 (medium)", "org": "Institute of Foundation Models", "settings": "Medium", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/k2-v2-medium", "metrics": { "index": 9.01595166538807 }, "details": { "Publisher model slug": "k2-v2-medium", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 512000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:mistral-small-4-non-reasoning", "model": "Mistral Small 4 (non-reasoning)", "org": "Mistral", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/mistral-small-4-non-reasoning", "metrics": { "index": 8.99008610259603 }, "details": { "Publisher model slug": "mistral-small-4-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.15, "Output price USD / million tokens": 0.6, "Median output tokens / second": 170.74290050464, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:tri-21b-think-v0-5", "model": "Tri-21B-Think", "org": "Trillion Labs", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/tri-21b-think-v0-5", "metrics": { "index": 8.98704047108094 }, "details": { "Publisher model slug": "tri-21b-think-v0-5", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 32000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-oss-20b", "model": "gpt-oss-20b (high)", "org": "OpenAI", "settings": "High", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gpt-oss-20b", "metrics": { "index": 8.9675171856126 }, "details": { "Publisher model slug": "gpt-oss-20b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": 0.01055611917474499, "Input price USD / million tokens": 0.05, "Output price USD / million tokens": 0.22, "Median output tokens / second": 151.56506382509, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-e4b", "model": "Gemma 4 E4B", "org": "Google", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/gemma-4-e4b", "metrics": { "index": 8.90908441839731 }, "details": { "Publisher model slug": "gemma-4-e4b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": 0.008163276245907385, "Input price USD / million tokens": 0.02, "Output price USD / million tokens": 0.1, "Median output tokens / second": 40.5604071354965, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-3-nano-30b-a3b-reasoning", "model": "Nemotron 3 Nano", "org": "NVIDIA", "settings": "Reasoning", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-nano-30b-a3b-reasoning", "metrics": { "index": 8.8961440122188 }, "details": { "Publisher model slug": "nvidia-nemotron-3-nano-30b-a3b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": 0.01675160944475661, "Input price USD / million tokens": 0.05, "Output price USD / million tokens": 0.2, "Median output tokens / second": 234.620563017979, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:minicpm5-1b", "model": "MiniCPM5-1B", "org": "OpenBMB", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/minicpm5-1b", "metrics": { "index": 8.79664321370212 }, "details": { "Publisher model slug": "minicpm5-1b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:sarvam-105b", "model": "Sarvam 105B (high)", "org": "Sarvam", "settings": "High", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/sarvam-105b", "metrics": { "index": 8.7905508357489 }, "details": { "Publisher model slug": "sarvam-105b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.042, "Output price USD / million tokens": 0.17, "Median output tokens / second": 192.173194754851, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-lite", "model": "Nova 2.0 Lite (non-reasoning)", "org": "Amazon", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-lite", "metrics": { "index": 8.73661367769335 }, "details": { "Publisher model slug": "nova-2-0-lite", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.3, "Output price USD / million tokens": 2.5, "Median output tokens / second": 175.97901794053, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:minicpm5-1b-non-reasoning", "model": "MiniCPM5-1B (non-reasoning)", "org": "OpenBMB", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/minicpm5-1b-non-reasoning", "metrics": { "index": 8.69473818567247 }, "details": { "Publisher model slug": "minicpm5-1b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:magistral-small-2509", "model": "Magistral Small 1.2", "org": "Mistral", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/magistral-small-2509", "metrics": { "index": 8.59366235978288 }, "details": { "Publisher model slug": "magistral-small-2509", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.5, "Output price USD / million tokens": 1.5, "Median output tokens / second": 78.1828676604879, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nanbeige4-1-3b", "model": "Nanbeige4.1-3B", "org": "Nanbeige", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nanbeige4-1-3b", "metrics": { "index": 8.4012282586083 }, "details": { "Publisher model slug": "nanbeige4-1-3b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:lfm2-5-2-6b", "model": "LFM2.5-2.6B", "org": "Liquid AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/lfm2-5-2-6b", "metrics": { "index": 8.38708534765242 }, "details": { "Publisher model slug": "lfm2-5-2-6b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:exaone-4-0-32b-reasoning", "model": "EXAONE 4.0 32B", "org": "LG AI Research", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/exaone-4-0-32b-reasoning", "metrics": { "index": 8.17836973273723 }, "details": { "Publisher model slug": "exaone-4-0-32b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-2-0-omni", "model": "Nova 2.0 Omni (non-reasoning)", "org": "Amazon", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-2-0-omni", "metrics": { "index": 8.15087501101863 }, "details": { "Publisher model slug": "nova-2-0-omni", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:hermes-4-llama-3-1-70b-reasoning", "model": "Hermes 4 70B", "org": "Nous Research", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/hermes-4-llama-3-1-70b-reasoning", "metrics": { "index": 7.90579275030455 }, "details": { "Publisher model slug": "hermes-4-llama-3-1-70b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:falcon-h1r-7b", "model": "Falcon-H1R-7B", "org": "TII UAE", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/falcon-h1r-7b", "metrics": { "index": 7.82776928077145 }, "details": { "Publisher model slug": "falcon-h1r-7b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-e2b", "model": "Gemma 4 E2B", "org": "Google", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-e2b", "metrics": { "index": 7.77445525187484 }, "details": { "Publisher model slug": "gemma-4-e2b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-omni-30b-a3b-reasoning", "model": "Qwen3 Omni 30B A3B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-omni-30b-a3b-reasoning", "metrics": { "index": 7.76082961984319 }, "details": { "Publisher model slug": "qwen3-omni-30b-a3b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.25, "Output price USD / million tokens": 0.97, "Median output tokens / second": 102.466628822164, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:step-3-vl-10b", "model": "Step3 VL 10B", "org": "StepFun", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/step-3-vl-10b", "metrics": { "index": 7.69391371030502 }, "details": { "Publisher model slug": "step-3-vl-10b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:llama-3-3-instruct-70b", "model": "Llama 3.3 70B", "org": "Meta", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/llama-3-3-instruct-70b", "metrics": { "index": 7.66380859299009 }, "details": { "Publisher model slug": "llama-3-3-instruct-70b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.655, "Output price USD / million tokens": 0.715, "Median output tokens / second": 85.7120216258369, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:llama-3-1-nemotron-ultra-253b-v1-reasoning", "model": "Llama Nemotron Ultra", "org": "NVIDIA", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/llama-3-1-nemotron-ultra-253b-v1-reasoning", "metrics": { "index": 7.52746905624501 }, "details": { "Publisher model slug": "llama-3-1-nemotron-ultra-253b-v1-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ernie-4-5-300b-a47b", "model": "ERNIE 4.5 300B A47B", "org": "Baidu", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/ernie-4-5-300b-a47b", "metrics": { "index": 7.50448042633837 }, "details": { "Publisher model slug": "ernie-4-5-300b-a47b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:hermes-4-llama-3-1-405b-reasoning", "model": "Hermes 4 405B", "org": "Nous Research", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/hermes-4-llama-3-1-405b-reasoning", "metrics": { "index": 7.49497809925135 }, "details": { "Publisher model slug": "hermes-4-llama-3-1-405b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 1, "Output price USD / million tokens": 3, "Median output tokens / second": 40.9558840666286, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-nano-12b-v2-vl-reasoning", "model": "NVIDIA Nemotron Nano 12B v2 VL", "org": "NVIDIA", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-nano-12b-v2-vl-reasoning", "metrics": { "index": 7.47796854824097 }, "details": { "Publisher model slug": "nvidia-nemotron-nano-12b-v2-vl-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-e4b-non-reasoning", "model": "Gemma 4 E4B (non-reasoning)", "org": "Google", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-e4b-non-reasoning", "metrics": { "index": 7.45765296912625 }, "details": { "Publisher model slug": "gemma-4-e4b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.02, "Output price USD / million tokens": 0.1, "Median output tokens / second": 49.0366613870711, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-nano-9b-v2-reasoning", "model": "NVIDIA Nemotron Nano 9B V2", "org": "NVIDIA", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-nano-9b-v2-reasoning", "metrics": { "index": 7.42882284148722 }, "details": { "Publisher model slug": "nvidia-nemotron-nano-9b-v2-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.04, "Output price USD / million tokens": 0.16, "Median output tokens / second": 95.0787111965213, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:hermes-4-llama-3-1-405b", "model": "Hermes 4 405B (non-reasoning)", "org": "Nous Research", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/hermes-4-llama-3-1-405b", "metrics": { "index": 7.41221110231024 }, "details": { "Publisher model slug": "hermes-4-llama-3-1-405b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 1, "Output price USD / million tokens": 3, "Median output tokens / second": 40.1699287728651, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-v2-low", "model": "K2-V2 (low)", "org": "Institute of Foundation Models", "settings": "Low", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/k2-v2-low", "metrics": { "index": 7.31130007284626 }, "details": { "Publisher model slug": "k2-v2-low", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 512000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:kimi-linear-48b-a3b-instruct", "model": "Kimi Linear 48B A3B Instruct", "org": "Kimi", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/kimi-linear-48b-a3b-instruct", "metrics": { "index": 7.30037362771236 }, "details": { "Publisher model slug": "kimi-linear-48b-a3b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:llama-3-1-instruct-405b", "model": "Llama 3.1 405B", "org": "Meta", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/llama-3-1-instruct-405b", "metrics": { "index": 7.2893092121128 }, "details": { "Publisher model slug": "llama-3-1-instruct-405b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:lfm2-5-8b-a1b", "model": "LFM2.5-8B-A1B", "org": "Liquid AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/lfm2-5-8b-a1b", "metrics": { "index": 7.21643567343487 }, "details": { "Publisher model slug": "lfm2-5-8b-a1b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 32768, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ring-flash-2-0", "model": "Ring-flash-2.0", "org": "InclusionAI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/ring-flash-2-0", "metrics": { "index": 7.15340447175535 }, "details": { "Publisher model slug": "ring-flash-2-0", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:olmo-3-1-32b-think", "model": "Olmo 3.1 32B Think", "org": "Allen Institute for AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/olmo-3-1-32b-think", "metrics": { "index": 7.12364412279906 }, "details": { "Publisher model slug": "olmo-3-1-32b-think", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 65500, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:command-a", "model": "Command A", "org": "Cohere", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/command-a", "metrics": { "index": 6.9581988704518 }, "details": { "Publisher model slug": "command-a", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": 2.5, "Output price USD / million tokens": 10, "Median output tokens / second": 50.5124132479221, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-2b", "model": "Qwen3.5 2B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-2b", "metrics": { "index": 6.94413664133259 }, "details": { "Publisher model slug": "qwen3-5-2b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-3-nano-30b-a3b", "model": "Nemotron 3 Nano (non-reasoning)", "org": "NVIDIA", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-nano-30b-a3b", "metrics": { "index": 6.8478519055181 }, "details": { "Publisher model slug": "nvidia-nemotron-3-nano-30b-a3b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 1000000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.05, "Output price USD / million tokens": 0.2, "Median output tokens / second": 268.446602538725, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-nano-9b-v2", "model": "NVIDIA Nemotron Nano 9B V2 (non-reasoning)", "org": "NVIDIA", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-nano-9b-v2", "metrics": { "index": 6.84412514839364 }, "details": { "Publisher model slug": "nvidia-nemotron-nano-9b-v2", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.05, "Output price USD / million tokens": 0.195, "Median output tokens / second": 165.256101837155, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:hermes-4-llama-3-1-70b", "model": "Hermes 4 70B (non-reasoning)", "org": "Nous Research", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/hermes-4-llama-3-1-70b", "metrics": { "index": 6.65822068462287 }, "details": { "Publisher model slug": "hermes-4-llama-3-1-70b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:sarvam-30b", "model": "Sarvam 30B (high)", "org": "Sarvam", "settings": "High", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/sarvam-30b", "metrics": { "index": 6.55971145066503 }, "details": { "Publisher model slug": "sarvam-30b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:olmo-3-1-32b-instruct", "model": "Olmo 3.1 32B Instruct", "org": "Allen Institute for AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/olmo-3-1-32b-instruct", "metrics": { "index": 6.49671923993574 }, "details": { "Publisher model slug": "olmo-3-1-32b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-4-e2b-non-reasoning", "model": "Gemma 4 E2B (non-reasoning)", "org": "Google", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-4-e2b-non-reasoning", "metrics": { "index": 6.47543520107148 }, "details": { "Publisher model slug": "gemma-4-e2b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:llama-4-scout", "model": "Llama 4 Scout", "org": "Meta", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/llama-4-scout", "metrics": { "index": 6.45126865234633 }, "details": { "Publisher model slug": "llama-4-scout", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 10000000, "Cost per index task (USD)": 0.1121761076625774, "Input price USD / million tokens": 0.18, "Output price USD / million tokens": 0.66, "Median output tokens / second": 102.657229438512, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:r1-1776", "model": "R1 1776", "org": "Perplexity", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/r1-1776", "metrics": { "index": 6.43857605749744 }, "details": { "Publisher model slug": "r1-1776", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:llama-3-2-instruct-90b-vision", "model": "Llama 3.2 90B (Vision)", "org": "Meta", "settings": "Vision", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/llama-3-2-instruct-90b-vision", "metrics": { "index": 6.4088295766243 }, "details": { "Publisher model slug": "llama-3-2-instruct-90b-vision", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:celeris-1", "model": "Celeris-1", "org": "Celeris", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/celeris-1", "metrics": { "index": 6.3469533538601 }, "details": { "Publisher model slug": "celeris-1", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 131072, "Cost per index task (USD)": 0.05015755586591194, "Input price USD / million tokens": 0.2, "Output price USD / million tokens": 0.7, "Median output tokens / second": 1580.64562591139, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:phi-4-mini", "model": "Phi-4 Mini", "org": "Microsoft", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/phi-4-mini", "metrics": { "index": 6.32860508899613 }, "details": { "Publisher model slug": "phi-4-mini", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.075, "Output price USD / million tokens": 0.3, "Median output tokens / second": 45.6172165677031, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:exaone-4-0-32b", "model": "EXAONE 4.0 32B (non-reasoning)", "org": "LG AI Research", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/exaone-4-0-32b", "metrics": { "index": 6.32621718534092 }, "details": { "Publisher model slug": "exaone-4-0-32b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 131000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-2b-non-reasoning", "model": "Qwen3.5 2B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-2b-non-reasoning", "metrics": { "index": 6.18559190112645 }, "details": { "Publisher model slug": "qwen3-5-2b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-0-8b", "model": "Qwen3.5 0.8B", "org": "Alibaba", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-0-8b", "metrics": { "index": 6.12847840160203 }, "details": { "Publisher model slug": "qwen3-5-0-8b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:deephermes-3-mistral-24b-preview", "model": "DeepHermes 3 - Mistral 24B (non-reasoning)", "org": "Nous Research", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/deephermes-3-mistral-24b-preview", "metrics": { "index": 6.07426212371414 }, "details": { "Publisher model slug": "deephermes-3-mistral-24b-preview", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 32000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:jamba-1-7-large", "model": "Jamba 1.7 Large", "org": "AI21 Labs", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/jamba-1-7-large", "metrics": { "index": 6.0710365237197 }, "details": { "Publisher model slug": "jamba-1-7-large", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-0-h-small", "model": "Granite 4.0 H Small", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/granite-4-0-h-small", "metrics": { "index": 6.04930220974964 }, "details": { "Publisher model slug": "granite-4-0-h-small", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ministral-3-14b", "model": "Ministral 3 14B", "org": "Mistral", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ministral-3-14b", "metrics": { "index": 6.04436817215002 }, "details": { "Publisher model slug": "ministral-3-14b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.01943640228162961, "Input price USD / million tokens": 0.2, "Output price USD / million tokens": 0.2, "Median output tokens / second": 83.0008292127636, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-omni-30b-a3b-instruct", "model": "Qwen3 Omni 30B A3B", "org": "Alibaba", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-omni-30b-a3b-instruct", "metrics": { "index": 6.00611816793457 }, "details": { "Publisher model slug": "qwen3-omni-30b-a3b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.25, "Output price USD / million tokens": 0.97, "Median output tokens / second": 96.3117395645235, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:lfm2-24b-a2b", "model": "LFM2 24B A2B", "org": "Liquid AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/lfm2-24b-a2b", "metrics": { "index": 5.94753271655266 }, "details": { "Publisher model slug": "lfm2-24b-a2b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 32768, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:phi-4", "model": "Phi-4", "org": "Microsoft", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/phi-4", "metrics": { "index": 5.92257028113751 }, "details": { "Publisher model slug": "phi-4", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 16000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.125, "Output price USD / million tokens": 0.5, "Median output tokens / second": 37.9947443448887, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nova-micro", "model": "Nova Micro", "org": "Amazon", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nova-micro", "metrics": { "index": 5.8765232381273 }, "details": { "Publisher model slug": "nova-micro", "Index version": "4.3.2", "Reasoning model": false, "Open weights": false, "Context window tokens": 130000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.035, "Output price USD / million tokens": 0.14, "Median output tokens / second": 279.741865000502, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-nano-12b-v2-vl", "model": "NVIDIA Nemotron Nano 12B v2 VL (non-reasoning)", "org": "NVIDIA", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-nano-12b-v2-vl", "metrics": { "index": 5.82045753777258 }, "details": { "Publisher model slug": "nvidia-nemotron-nano-12b-v2-vl", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.2, "Output price USD / million tokens": 0.6, "Median output tokens / second": 202.436005960036, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:phi-4-multimodal", "model": "Phi-4 Multimodal", "org": "Microsoft", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/phi-4-multimodal", "metrics": { "index": 5.80520041934726 }, "details": { "Publisher model slug": "phi-4-multimodal", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.08, "Output price USD / million tokens": 0.32, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:minicpm-v4-6-1-3b", "model": "MiniCPM-V 4.6 1.3B", "org": "OpenBMB", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/minicpm-v4-6-1-3b", "metrics": { "index": 5.69026588282761 }, "details": { "Publisher model slug": "minicpm-v4-6-1-3b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:jamba-reasoning-3b", "model": "Jamba Reasoning 3B", "org": "AI21 Labs", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/jamba-reasoning-3b", "metrics": { "index": 5.66829948484426 }, "details": { "Publisher model slug": "jamba-reasoning-3b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:reka-flash-3", "model": "Reka Flash 3", "org": "Reka AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/reka-flash-3", "metrics": { "index": 5.64602230491152 }, "details": { "Publisher model slug": "reka-flash-3", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.2, "Output price USD / million tokens": 0.8, "Median output tokens / second": 88.0242860874207, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:olmo-3-7b-think", "model": "Olmo 3 7B Think", "org": "Allen Institute for AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/olmo-3-7b-think", "metrics": { "index": 5.61691854830355 }, "details": { "Publisher model slug": "olmo-3-7b-think", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:molmo-7b-d", "model": "Molmo 7B-D", "org": "Allen Institute for AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/molmo-7b-d", "metrics": { "index": 5.56262049746202 }, "details": { "Publisher model slug": "molmo-7b-d", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 4096, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ministral-3-8b", "model": "Ministral 3 8B", "org": "Mistral", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ministral-3-8b", "metrics": { "index": 5.47767370454416 }, "details": { "Publisher model slug": "ministral-3-8b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.011086416592507111, "Input price USD / million tokens": 0.15, "Output price USD / million tokens": 0.15, "Median output tokens / second": 113.104034471304, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:llama-3-2-instruct-11b-vision", "model": "Llama 3.2 11B (Vision)", "org": "Meta", "settings": "Vision", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/llama-3-2-instruct-11b-vision", "metrics": { "index": 5.40802026647316 }, "details": { "Publisher model slug": "llama-3-2-instruct-11b-vision", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.345, "Output price USD / million tokens": 0.345, "Median output tokens / second": 22.8725995293842, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:qwen3-5-0-8b-non-reasoning", "model": "Qwen3.5 0.8B (non-reasoning)", "org": "Alibaba", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/qwen3-5-0-8b-non-reasoning", "metrics": { "index": 5.40008099703412 }, "details": { "Publisher model slug": "qwen3-5-0-8b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:exaone-4-0-1-2b-reasoning", "model": "Exaone 4.0 1.2B", "org": "LG AI Research", "settings": "Reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/exaone-4-0-1-2b-reasoning", "metrics": { "index": 5.27138994054015 }, "details": { "Publisher model slug": "exaone-4-0-1-2b-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 64000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:olmo-3-7b-instruct", "model": "Olmo 3 7B", "org": "Allen Institute for AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/olmo-3-7b-instruct", "metrics": { "index": 5.24099718758815 }, "details": { "Publisher model slug": "olmo-3-7b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:exaone-4-0-1-2b", "model": "Exaone 4.0 1.2B (non-reasoning)", "org": "LG AI Research", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/exaone-4-0-1-2b", "metrics": { "index": 5.22938457197929 }, "details": { "Publisher model slug": "exaone-4-0-1-2b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 64000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:lfm2-5-1-2b-thinking", "model": "LFM2.5-1.2B-Thinking", "org": "Liquid AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/lfm2-5-1-2b-thinking", "metrics": { "index": 5.22188553175398 }, "details": { "Publisher model slug": "lfm2-5-1-2b-thinking", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 32000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:jamba-1-7-mini", "model": "Jamba 1.7 Mini", "org": "AI21 Labs", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/jamba-1-7-mini", "metrics": { "index": 5.21767706986744 }, "details": { "Publisher model slug": "jamba-1-7-mini", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 258000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:lfm2-5-1-2b-instruct", "model": "LFM2.5-1.2B-Instruct", "org": "Liquid AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/lfm2-5-1-2b-instruct", "metrics": { "index": 5.20951181661378 }, "details": { "Publisher model slug": "lfm2-5-1-2b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 32000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-0-h-nano-1b", "model": "Granite 4.0 H 1B", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/granite-4-0-h-nano-1b", "metrics": { "index": 5.1945517329943 }, "details": { "Publisher model slug": "granite-4-0-h-nano-1b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemma-3-270m", "model": "Gemma 3 270M", "org": "Google", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/gemma-3-270m", "metrics": { "index": 5.11705960692171 }, "details": { "Publisher model slug": "gemma-3-270m", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 32000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:apertus-70b-instruct", "model": "Apertus 70B Instruct", "org": "Swiss AI Initiative", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/apertus-70b-instruct", "metrics": { "index": 5.11462775284761 }, "details": { "Publisher model slug": "apertus-70b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-0-micro", "model": "Granite 4.0 Micro", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/granite-4-0-micro", "metrics": { "index": 5.10657830942846 }, "details": { "Publisher model slug": "granite-4-0-micro", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:deephermes-3-llama-3-1-8b-preview", "model": "DeepHermes 3 - Llama-3.1 8B (non-reasoning)", "org": "Nous Research", "settings": "Non-reasoning", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/deephermes-3-llama-3-1-8b-preview", "metrics": { "index": 5.08032603461277 }, "details": { "Publisher model slug": "deephermes-3-llama-3-1-8b-preview", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:molmo2-8b", "model": "Molmo2-8B", "org": "Allen Institute for AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/molmo2-8b", "metrics": { "index": 5.00251209304105 }, "details": { "Publisher model slug": "molmo2-8b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 36864, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:ministral-3-3b", "model": "Ministral 3 3B", "org": "Mistral", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/ministral-3-3b", "metrics": { "index": 4.83760020546443 }, "details": { "Publisher model slug": "ministral-3-3b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 256000, "Cost per index task (USD)": 0.007846860809830004, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.1, "Median output tokens / second": 207.33418018142, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:apertus-8b-instruct", "model": "Apertus 8B Instruct", "org": "Swiss AI Initiative", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/apertus-8b-instruct", "metrics": { "index": 4.83423948 }, "details": { "Publisher model slug": "apertus-8b-instruct", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 65536, "Cost per index task (USD)": null, "Input price USD / million tokens": 0.1, "Output price USD / million tokens": 0.2, "Median output tokens / second": 130.853930915026, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-0-350m", "model": "Granite 4.0 350M", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/granite-4-0-350m", "metrics": { "index": 4.83423948 }, "details": { "Publisher model slug": "granite-4-0-350m", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 32768, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:granite-4-0-h-350m", "model": "Granite 4.0 H 350M", "org": "IBM", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/granite-4-0-h-350m", "metrics": { "index": 4.83423948 }, "details": { "Publisher model slug": "granite-4-0-h-350m", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 32768, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:lfm2-5-vl-1-6b", "model": "LFM2.5-VL-1.6B", "org": "Liquid AI", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/lfm2-5-vl-1-6b", "metrics": { "index": 4.83423948 }, "details": { "Publisher model slug": "lfm2-5-vl-1-6b", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 32000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:tiny-aya-global", "model": "Tiny Aya Global", "org": "Cohere", "settings": "Not specified beyond the source variant", "date": null, "estimated": true, "status": "Publisher estimate", "url": "https://artificialanalysis.ai/models/tiny-aya-global", "metrics": { "index": 4.83423948 }, "details": { "Publisher model slug": "tiny-aya-global", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 8192, "Cost per index task (USD)": null, "Input price USD / million tokens": 0, "Output price USD / million tokens": 0, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nvidia-nemotron-3-nano-4b", "model": "Nemotron 3 Nano 4B", "org": "NVIDIA", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/nvidia-nemotron-3-nano-4b", "metrics": { "index": 4.76910061591993 }, "details": { "Publisher model slug": "nvidia-nemotron-3-nano-4b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 262000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:k2-horizon-0-9b", "model": "K2 Horizon 0.9B", "org": "Institute of Foundation Models", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "Independently measured", "url": "https://artificialanalysis.ai/models/k2-horizon-0-9b", "metrics": { "index": 3.04760621054014 }, "details": { "Publisher model slug": "k2-horizon-0-9b", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:cogito-v2-1-reasoning", "model": "Cogito v2.1", "org": "Deep Cogito", "settings": "Reasoning", "date": null, "estimated": false, "status": "No published index", "url": "https://artificialanalysis.ai/models/cogito-v2-1-reasoning", "metrics": { "index": null }, "details": { "Publisher model slug": "cogito-v2-1-reasoning", "Index version": "4.3.2", "Reasoning model": true, "Open weights": true, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:exaone-4-5-33b-non-reasoning", "model": "EXAONE 4.5 33B (non-reasoning)", "org": "LG AI Research", "settings": "Non-reasoning", "date": null, "estimated": false, "status": "No published index", "url": "https://artificialanalysis.ai/models/exaone-4-5-33b-non-reasoning", "metrics": { "index": null }, "details": { "Publisher model slug": "exaone-4-5-33b-non-reasoning", "Index version": "4.3.2", "Reasoning model": false, "Open weights": true, "Context window tokens": 262144, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-5-5-pro", "model": "GPT-5.5 Pro (xhigh)", "org": "OpenAI", "settings": "Xhigh", "date": null, "estimated": false, "status": "No published index", "url": "https://artificialanalysis.ai/models/gpt-5-5-pro", "metrics": { "index": null }, "details": { "Publisher model slug": "gpt-5-5-pro", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 922000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gpt-6-sol-daybreak-blue", "model": "GPT-6 Sol (Daybreak Blue, max)", "org": "OpenAI", "settings": "Daybreak Blue, max", "date": null, "estimated": false, "status": "No published index", "url": "https://artificialanalysis.ai/models/gpt-6-sol-daybreak-blue", "metrics": { "index": null }, "details": { "Publisher model slug": "gpt-6-sol-daybreak-blue", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 1050000, "Cost per index task (USD)": null, "Input price USD / million tokens": 2, "Output price USD / million tokens": 10, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:gemini-3-deep-think", "model": "Gemini 3 Deep Think", "org": "Google", "settings": "Not specified beyond the source variant", "date": null, "estimated": false, "status": "No published index", "url": "https://artificialanalysis.ai/models/gemini-3-deep-think", "metrics": { "index": null }, "details": { "Publisher model slug": "gemini-3-deep-think", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 128000, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } }, { "id": "aa:nemotron-3-5-bolt-e4b-4gb", "model": "Nemotron 3.5 Bolt E4B 4GB", "org": "NVIDIA", "settings": "Reasoning", "date": null, "estimated": false, "status": "No published index", "url": "https://artificialanalysis.ai/models/nemotron-3-5-bolt-e4b-4gb", "metrics": { "index": null }, "details": { "Publisher model slug": "nemotron-3-5-bolt-e4b-4gb", "Index version": "4.3.2", "Reasoning model": true, "Open weights": false, "Context window tokens": 131072, "Cost per index task (USD)": null, "Input price USD / million tokens": null, "Output price USD / million tokens": null, "Median output tokens / second": null, "Uncertainty": "Publisher estimate flag retained; no statistical interval supplied" } } ] }, { "id": "swebench", "label": "SWE-bench Verified · Bash Only", "url": "https://www.swebench.com/", "methodologyUrl": "https://www.swebench.com/verified.html", "asOf": "Runs 2025-07-20 to 2026-02-26", "retrievedAt": "2026-10-25T04:30:11Z", "description": "Supplementary repository issue-resolution evidence. Select a mini-SWE-agent harness version: v1 and v2 are not necessarily comparable. The newest available version is selected by default, and Clear filters restores it.", "coverage": { "runs": 47, "publisher_checked": 30 }, "caveats": [ "Retrieval date is not an evaluation date; individual run dates are retained.", "Model API identifiers, reasoning settings and harness versions are not interchangeable.", "Null reasoning effort means not specified by this dataset, not no reasoning.", "Null publisher_checked means not explicitly marked; it must not be converted to true or false.", "Scores depend on mini-SWE-agent version and configuration; compare like configurations.", "Publisher warns v1.x and v2.x are not necessarily comparable: v2 uses tool calls, v1 parses action strings, and temperature settings differ.", "SWE-bench Verified is a static public 500-task benchmark, with contamination and saturation risks.", "Do not count this dataset again through vendor SWE-bench Verified results or an overlapping composite.", "Different benchmark variants (Verified, Pro, Lite, Multilingual) must not be merged as one measurement.", "The benchmark contains 500 tasks; run-level sample sizes, repetitions and statistical intervals are not provided in these selected fields and remain null.", "The published run dates span 20 July 2025 to 26 February 2026. Retrieval time is not an evaluation date.", "Harness v1.x/v2.x use different interaction/temperature settings and must not be treated as one controlled comparison: https://www.swebench.com/verified.html" ], "defaultHarness": "2.0.0", "metrics": [ { "key": "resolved", "label": "Resolved", "unit": "% of issues" } ], "rows": [ { "id": "20260217_mini-v2.0.0_claude-4-5-opus-high", "model": "Claude 4.5 Opus (high)", "org": "Anthropic", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 76.8 }, "details": { "API model identifiers": [ "claude-4-5-opus" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: claude-4-5-opus", "Org: Anthropic", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/b038912e-0133-4594-b093-92806f8ffb17", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_gemini-3-flash-high", "model": "Gemini 3 Flash (high)", "org": "Google DeepMind", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 75.8 }, "details": { "API model identifiers": [ "gemini-3-flash-preview" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: gemini-3-flash-preview", "Org: Google DeepMind", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/1ebbdd7a-55b3-4015-9b83-5978cc7fb618", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_minimax-2-5-high", "model": "MiniMax M2.5 (high)", "org": "Minimax", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 75.8 }, "details": { "API model identifiers": [ "minimax-m2.5" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: minimax-m2.5", "Org: Minimax", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/5b77e003-7328-4003-879e-9b55dd3a0b6f", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_claude-4-6-opus", "model": "Claude 4.6 Opus", "org": "Anthropic", "settings": "mini-SWE-agent 2.0.0; effort: not published", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 75.6 }, "details": { "API model identifiers": [ "claude-opus-4-6" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": null, "Configuration tags": [ "Model: claude-opus-4-6", "Org: Anthropic", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/9243cc78-d399-402f-be97-e366ff63282c", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251124_mini-v1.16.0_claude-opus-4-5-20251101", "model": "Claude 4.5 Opus (20251101) (medium)", "org": "Anthropic", "settings": "mini-SWE-agent 1.16.0; effort: medium", "harness": "1.16.0", "date": "2025-11-24", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 74.4 }, "details": { "API model identifiers": [ "claude-opus-4-5-20251101" ], "Harness version": "1.16.0", "Harness major version": "1", "Reasoning effort": "medium", "Configuration tags": [ "Model: claude-opus-4-5-20251101", "Org: Anthropic", "System: Attempts - 1", "Mini: 1.16.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/6f6e2390-00c7-479f-9a2a-87b53b7477cb", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251118_mini-v1.15.0_gemini-3-pro-preview-20251118", "model": "Gemini 3 Pro Preview (2025-11-18)", "org": "Google DeepMind", "settings": "mini-SWE-agent 1.15.0; effort: not published", "harness": "1.15.0", "date": "2025-11-18", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 74.2 }, "details": { "API model identifiers": [ "gemini-3-pro-preview" ], "Harness version": "1.15.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: gemini-3-pro-preview", "Org: Google DeepMind", "System: Attempts - 1", "Mini: 1.15.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/3641b17f-034e-4b36-aa66-471dfed837d6", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_glm-5-high", "model": "GLM 5 (high)", "org": "Z-AI", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 72.8 }, "details": { "API model identifiers": [ "glm-5" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: glm-5", "Org: Z-AI", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/5fc22eb4-1f44-405e-9dff-40a61f9f6ef6", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_gpt-5-2-high", "model": "GPT 5.2 (high)", "org": "OpenAI", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 72.8 }, "details": { "API model identifiers": [ "gpt-5-2" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: gpt-5-2", "Org: OpenAI", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/c2c7f5be-2c1a-472f-a34a-92e9e1df4f01", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260219_mini-v2.0.0_gpt-5-2-codex", "model": "GPT 5.2 Codex", "org": "OpenAI", "settings": "mini-SWE-agent 2.0.0; effort: not published", "harness": "2.0.0", "date": "2026-02-19", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 72.8 }, "details": { "API model identifiers": [ "gpt-5-2-codex" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": null, "Configuration tags": [ "Model: gpt-5-2-codex", "Org: OpenAI", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/fb22a2e4-0a41-4d41-8e1e-388d4cb50d80", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high", "model": "GPT 5.2 (2025-12-11) (high)", "org": "OpenAI", "settings": "mini-SWE-agent 1.17.2; effort: high", "harness": "1.17.2", "date": "2025-12-11", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 71.8 }, "details": { "API model identifiers": [ "gpt-5.2-2025-12-11" ], "Harness version": "1.17.2", "Harness major version": "1", "Reasoning effort": "high", "Configuration tags": [ "Model: gpt-5.2-2025-12-11", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.17.2" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/93ef4179-a475-4f0b-88d1-46b5aa75e19b", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_claude-4-5-sonnet-high", "model": "Claude 4.5 Sonnet (high)", "org": "Anthropic", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 71.4 }, "details": { "API model identifiers": [ "claude-sonnet-4-5-20250929" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: claude-sonnet-4-5-20250929", "Org: Anthropic", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/737e5dd2-8555-435a-9fbd-1c6907c972f1", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_kimi-k2-5-high", "model": "Kimi K2.5 (high)", "org": "Moonshot AI", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 70.8 }, "details": { "API model identifiers": [ "kimi-k2.5" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: kimi-k2.5", "Org: Moonshot AI", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/c4f6e550-6e49-436e-96fa-3aafd9a0c936", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250929_mini-v1.13.3_sonnet-4-5-20250929", "model": "Claude 4.5 Sonnet (20250929)", "org": "Anthropic", "settings": "mini-SWE-agent 1.13.3; effort: not published", "harness": "1.13.3", "date": "2025-09-29", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 70.6 }, "details": { "API model identifiers": [ "claude-sonnet-4-5-20250929" ], "Harness version": "1.13.3", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: claude-sonnet-4-5-20250929", "Org: Anthropic", "System: Attempts - 1", "Mini: 1.13.3" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/a4844da1-fbb9-4d61-b82c-f46e471f748a", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_deepseek-3-2-high", "model": "DeepSeek V3.2 (high)", "org": "DeepSeek", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 70.0 }, "details": { "API model identifiers": [ "deepseek-v3.2" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: deepseek-v3.2", "Org: DeepSeek", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/4b463c14-4776-49a2-accf-b42b1c076bba", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260226_mini-v2.0.0_gemini-3-pro-high", "model": "Gemini 3 Pro", "org": "Google DeepMind", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-26", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 69.6 }, "details": { "API model identifiers": [ "gemini-3-pro-preview" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: gemini-3-pro-preview", "Org: Google DeepMind", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/cf47db93-89ae-4f15-984d-e117c77cb96d", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251211_mini-v1.17.2_gpt-5.2-2025-12-11", "model": "GPT 5.2 (2025-12-11)", "org": "OpenAI", "settings": "mini-SWE-agent 1.17.2; effort: not published", "harness": "1.17.2", "date": "2025-12-11", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 69.0 }, "details": { "API model identifiers": [ "gpt-5.2-2025-12-11" ], "Harness version": "1.17.2", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: gpt-5.2-2025-12-11", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.17.2" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/c0d7b5a6-9d24-45d4-9e4b-3ae87f2c654a", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250802_mini-v1.0.0_claude-4-opus-20250514", "model": "Claude 4 Opus (20250514)", "org": "Anthropic", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-08-02", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 67.6 }, "details": { "API model identifiers": [ "claude-4-opus-20250514" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: claude-4-opus-20250514", "Org: Anthropic", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/c93d3381-a539-4cd6-89be-97066d36d906", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_claude-4-5-haiku-high", "model": "Claude 4.5 Haiku (high)", "org": "Anthropic", "settings": "mini-SWE-agent 2.0.0; effort: high", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 66.6 }, "details": { "API model identifiers": [ "claude-haiku-4-5-20251001" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": "high", "Configuration tags": [ "Model: claude-haiku-4-5-20251001", "Org: Anthropic", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/a91fc32b-2035-4ec0-950e-433922852071", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251120_mini-v1.15.0_gpt-5.1-2025-11-13", "model": "GPT 5.1 (2025-11-13) (medium)", "org": "OpenAI", "settings": "mini-SWE-agent 1.15.0; effort: medium", "harness": "1.15.0", "date": "2025-11-20", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 66.0 }, "details": { "API model identifiers": [ "gpt-5.1-2025-11-13" ], "Harness version": "1.15.0", "Harness major version": "1", "Reasoning effort": "medium", "Configuration tags": [ "Model: gpt-5.1-2025-11-13", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.15.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/0f6398cb-461e-478a-8030-85b2d005d987", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251124_mini-v1.16.0_gpt-5.1-codex", "model": "GPT 5.1 Codex (medium)", "org": "OpenAI", "settings": "mini-SWE-agent 1.16.0; effort: medium", "harness": "1.16.0", "date": "2025-11-24", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 66.0 }, "details": { "API model identifiers": [ "gpt-5.1-codex" ], "Harness version": "1.16.0", "Harness major version": "1", "Reasoning effort": "medium", "Configuration tags": [ "Model: gpt-5.1-codex", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.16.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/ebeb5896-b0bc-4504-a687-a5bf9824a146", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250807_mini-v1.7.0_gpt-5", "model": "GPT 5 (2025-08-07) (medium)", "org": "OpenAI", "settings": "mini-SWE-agent 1.7.0; effort: medium", "harness": "1.7.0", "date": "2025-08-07", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 65.0 }, "details": { "API model identifiers": [ "gpt-5-2025-08-07" ], "Harness version": "1.7.0", "Harness major version": "1", "Reasoning effort": "medium", "Configuration tags": [ "Model: gpt-5-2025-08-07", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.7.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/4844cb3c-d5cd-4c75-b64b-973029f2d37c", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250726_mini-v1.0.0_claude-sonnet-4-20250514", "model": "Claude 4 Sonnet (20250514)", "org": "Anthropic", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-07-26", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 64.93 }, "details": { "API model identifiers": [ "claude-4-sonnet-20250514" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: claude-4-sonnet-20250514", "Org: Anthropic", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/0cb59666-bca8-476b-bf8e-3b924fafcae7", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251210_mini-v1.17.2_kimi-k2-thinking", "model": "Kimi K2 Thinking", "org": "Moonshot AI", "settings": "mini-SWE-agent 1.17.2; effort: not published", "harness": "1.17.2", "date": "2025-12-10", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 63.4 }, "details": { "API model identifiers": [ "Kimi-K2-Thinking" ], "Harness version": "1.17.2", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: Kimi-K2-Thinking", "Org: Moonshot AI", "System: Attempts - 1", "Mini: 1.17.2" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/9dcb7016-5356-4966-a184-ce4b1131f33c", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251124_mini-v1.17.0_minimax-m2", "model": "MiniMax M2", "org": "Minimax", "settings": "mini-SWE-agent 1.17.0; effort: not published", "harness": "1.17.0", "date": "2025-11-24", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 61.0 }, "details": { "API model identifiers": [ "minimax-m2" ], "Harness version": "1.17.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: minimax-m2", "Org: Minimax", "System: Attempts - 1", "Mini: 1.17.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/dcfba689-d8e6-4372-9e92-dd524b7a0c56", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251201_mini-v1.17.1_deepseek-v3.2-reasoner", "model": "DeepSeek V3.2 Reasoner", "org": "deepseek", "settings": "mini-SWE-agent 1.17.1; effort: not published", "harness": "1.17.1", "date": "2025-12-01", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 60.0 }, "details": { "API model identifiers": [ "deepseek-v3.2-reasoner" ], "Harness version": "1.17.1", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: deepseek-v3.2-reasoner", "Org: deepseek", "System: Attempts - 1", "Mini: 1.17.1" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/7fde5552-6b17-4cb7-ab9c-15fd9fb5b845", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250807_mini-v1.7.0_gpt-5-mini", "model": "GPT 5 mini (2025-08-07) (medium)", "org": "OpenAI", "settings": "mini-SWE-agent 1.7.0; effort: medium", "harness": "1.7.0", "date": "2025-08-07", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 59.8 }, "details": { "API model identifiers": [ "gpt-5-mini-2025-08-07" ], "Harness version": "1.7.0", "Harness major version": "1", "Reasoning effort": "medium", "Configuration tags": [ "Model: gpt-5-mini-2025-08-07", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.7.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/7f5e3cfc-8e38-4c73-812c-aca21298e7bc", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250726_mini-v1.0.0_o3-2025-04-16", "model": "o3 (2025-04-16)", "org": "OpenAI", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-07-26", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 58.4 }, "details": { "API model identifiers": [ "o3-20250416" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: o3-20250416", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/565e5680-b913-4031-b537-00721a7a619a", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251209_mini-v1.17.2_devstral-small-2512", "model": "Devstral Small (2512)", "org": "mistral", "settings": "mini-SWE-agent 1.17.2; effort: not published", "harness": "1.17.2", "date": "2025-12-09", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 56.4 }, "details": { "API model identifiers": [ "devstral-small-2512" ], "Harness version": "1.17.2", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: devstral-small-2512", "Org: mistral", "System: Attempts - 1", "Mini: 1.17.2" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20260217_mini-v2.0.0_gpt-5-mini", "model": "GPT 5 mini", "org": "OpenAI", "settings": "mini-SWE-agent 2.0.0; effort: not published", "harness": "2.0.0", "date": "2026-02-17", "estimated": false, "status": "Checking not marked", "url": "https://www.swebench.com/", "metrics": { "resolved": 56.2 }, "details": { "API model identifiers": [ "gpt-5-mini-2025-08-07" ], "Harness version": "2.0.0", "Harness major version": "2", "Reasoning effort": null, "Configuration tags": [ "Model: gpt-5-mini-2025-08-07", "Org: OpenAI", "System: Attempts - 1", "Mini: 2.0.0" ], "Publisher checked": null, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/7e2c27bc-5b56-4359-b871-677bcb7bbdf1", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250802_mini-v1.0.0_qwen3-coder-480b-a35b-instruct", "model": "Qwen3-Coder 480B/A35B Instruct", "org": "Qwen", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-08-02", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 55.4 }, "details": { "API model identifiers": [ "Qwen3-Coder-480B-A35B-Instruct" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: Qwen3-Coder-480B-A35B-Instruct", "Org: Qwen", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/f39d3041-d9d7-4f1b-b75e-8a13addb9e6e", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251201_mini-v1.17.1_glm-4.6", "model": "GLM 4.6 (T=1)", "org": "Z.ai", "settings": "mini-SWE-agent 1.17.1; effort: not published", "harness": "1.17.1", "date": "2025-12-01", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 55.4 }, "details": { "API model identifiers": [ "glm-4.6" ], "Harness version": "1.17.1", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: glm-4.6", "Org: Z.ai", "System: Attempts - 1", "Mini: 1.17.1" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/1fc7d1bf-f715-42a1-8ebb-4dc19ccc8fc0", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250822_mini-v1.9.1_glm-4.5", "model": "GLM 4.5 (2025-08-22)", "org": "Z.ai", "settings": "mini-SWE-agent 1.9.1; effort: not published", "harness": "1.9.1", "date": "2025-08-22", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 54.2 }, "details": { "API model identifiers": [ "GLM-4.5" ], "Harness version": "1.9.1", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: GLM-4.5", "Org: Z.ai", "System: Attempts - 1", "Mini: 1.9.1" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/9ae7f9a3-c9e3-48af-833c-52adcc7f5921", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20251209_mini-v1.17.2_devstral-2512", "model": "Devstral (2512)", "org": "mistral", "settings": "mini-SWE-agent 1.17.2; effort: not published", "harness": "1.17.2", "date": "2025-12-09", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 53.8 }, "details": { "API model identifiers": [ "devstral-2512" ], "Harness version": "1.17.2", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: devstral-2512", "Org: mistral", "System: Attempts - 1", "Mini: 1.17.2" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250726_mini-v1.0.0_gemini-2.5-pro", "model": "Gemini 2.5 Pro (2025-05-06)", "org": "Google DeepMind", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-07-26", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 53.6 }, "details": { "API model identifiers": [ "gemini-2.5-pro" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: gemini-2.5-pro", "Org: Google DeepMind", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/8b22ae97-2cfd-4b05-8438-38550cfaa163", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250720_mini-v0.0.0-claude-3-7-sonnet-20250219", "model": "Claude 3.7 Sonnet (20250219)", "org": "Anthropic", "settings": "mini-SWE-agent 0.0.0; effort: not published", "harness": "0.0.0", "date": "2025-07-20", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 52.8 }, "details": { "API model identifiers": [ "claude-3-7-sonnet-20250219" ], "Harness version": "0.0.0", "Harness major version": "0", "Reasoning effort": null, "Configuration tags": [ "Model: claude-3-7-sonnet-20250219", "Org: Anthropic", "System: Attempts - 1", "Mini: 0.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250726_mini-v1.0.0_o4-mini-2025-04-16", "model": "o4-mini (2025-04-16)", "org": "OpenAI", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-07-26", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 45.0 }, "details": { "API model identifiers": [ "o4-mini-20250416" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: o4-mini-20250416", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/f46eb5b4-b4b2-478e-bc11-f1eb47eca637", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250807_mini-v1.7.0_kimi-k2-instruct", "model": "Kimi K2 Instruct", "org": "Moonshot AI", "settings": "mini-SWE-agent 1.7.0; effort: not published", "harness": "1.7.0", "date": "2025-08-07", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 43.8 }, "details": { "API model identifiers": [ "Kimi-K2-Instruct" ], "Harness version": "1.7.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: Kimi-K2-Instruct", "Org: Moonshot AI", "System: Attempts - 1", "Mini: 1.7.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/b6a6d69b-e69e-4011-aa94-46f44b84202e", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250726_mini-v1.0.0_gpt-4.1-2025-04-14", "model": "GPT 4.1 (2025-04-14)", "org": "OpenAI", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-07-26", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 39.58 }, "details": { "API model identifiers": [ "gpt-4.1-20250414" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: gpt-4.1-20250414", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/cd7a23c5-a2b1-4cab-b851-6e2c42aaf0f3", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250807_mini-v1.7.0_gpt-5-nano", "model": "GPT 5 nano (2025-08-07) (medium)", "org": "OpenAI", "settings": "mini-SWE-agent 1.7.0; effort: medium", "harness": "1.7.0", "date": "2025-08-07", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 34.8 }, "details": { "API model identifiers": [ "gpt-5-nano-2025-08-07" ], "Harness version": "1.7.0", "Harness major version": "1", "Reasoning effort": "medium", "Configuration tags": [ "Model: gpt-5-nano-2025-08-07", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.7.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/3dc9b424-547c-4bdc-9b72-3b58d316cb0a", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250726_mini-v1.0.0_gemini-2.5-flash", "model": "Gemini 2.5 Flash (2025-04-17)", "org": "Google DeepMind", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-07-26", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 28.73 }, "details": { "API model identifiers": [ "gemini-2.5-flash" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: gemini-2.5-flash", "Org: Google DeepMind", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/37e5933e-71a1-4011-8340-fbca563124b3", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250807_mini-v1.7.0_gpt-oss-120b", "model": "gpt-oss-120b", "org": "OpenAI", "settings": "mini-SWE-agent 1.7.0; effort: not published", "harness": "1.7.0", "date": "2025-08-07", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 26.0 }, "details": { "API model identifiers": [ "gpt-oss-120b" ], "Harness version": "1.7.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: gpt-oss-120b", "Org: OpenAI", "System: Attempts - 1", "Mini: 1.7.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/4a0a2b2a-a65b-452a-88c7-bb5d99745c75", "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250720_mini-v0.0.0_gpt-4.1-mini-2025-04-14", "model": "GPT 4.1 mini (2025-04-14)", "org": "OpenAI", "settings": "mini-SWE-agent 0.0.0; effort: not published", "harness": "0.0.0", "date": "2025-07-20", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 23.94 }, "details": { "API model identifiers": [ "gpt-4.1-mini-20250414" ], "Harness version": "0.0.0", "Harness major version": "0", "Reasoning effort": null, "Configuration tags": [ "Model: gpt-4.1-mini-20250414", "Org: OpenAI", "System: Attempts - 1", "Mini: 0.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250720_mini-v0.0.0-gpt-4o-2024-11-20", "model": "GPT 4o (2024-11-20)", "org": "OpenAI", "settings": "mini-SWE-agent 0.0.0; effort: not published", "harness": "0.0.0", "date": "2025-07-20", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 21.62 }, "details": { "API model identifiers": [ "gpt-4o-20241120" ], "Harness version": "0.0.0", "Harness major version": "0", "Reasoning effort": null, "Configuration tags": [ "Model: gpt-4o-20241120", "Org: OpenAI", "System: Attempts - 1", "Mini: 0.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250720_mini-v0.0.0-Llama-4-Maverick-17B-Instruct", "model": "Llama 4 Maverick Instruct", "org": "Meta", "settings": "mini-SWE-agent 0.0.0; effort: not published", "harness": "0.0.0", "date": "2025-07-20", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 21.04 }, "details": { "API model identifiers": [ "llama-4-maverick-instruct" ], "Harness version": "0.0.0", "Harness major version": "0", "Reasoning effort": null, "Configuration tags": [ "Model: llama-4-maverick-instruct", "Org: Meta", "System: Attempts - 1", "Mini: 0.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250726_mini-v1.0.0_gemini-2.0-flash", "model": "Gemini 2.0 Flash", "org": "Google DeepMind", "settings": "mini-SWE-agent 0.0.0; effort: not published", "harness": "0.0.0", "date": "2025-07-26", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 13.52 }, "details": { "API model identifiers": [ "gemini-2.0-flash" ], "Harness version": "0.0.0", "Harness major version": "0", "Reasoning effort": null, "Configuration tags": [ "Model: gemini-2.0-flash", "Org: Google DeepMind", "System: Attempts - 1", "Mini: 0.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250720_mini-v0.0.0-Llama-4-Scout-17B-Instruct", "model": "Llama 4 Scout Instruct", "org": "Meta", "settings": "mini-SWE-agent 0.0.0; effort: not published", "harness": "0.0.0", "date": "2025-07-20", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 9.06 }, "details": { "API model identifiers": [ "llama-4-scout-instruct" ], "Harness version": "0.0.0", "Harness major version": "0", "Reasoning effort": null, "Configuration tags": [ "Model: llama-4-scout-instruct", "Org: Meta", "System: Attempts - 1", "Mini: 0.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": null, "Uncertainty": "No publisher confidence interval supplied" } }, { "id": "20250803_mini-v1.0.0_qwen2-5-coder-32b-instruct", "model": "Qwen2.5-Coder 32B Instruct", "org": "Qwen", "settings": "mini-SWE-agent 1.0.0; effort: not published", "harness": "1.0.0", "date": "2025-08-03", "estimated": false, "status": "Publisher checked", "url": "https://www.swebench.com/", "metrics": { "resolved": 9.0 }, "details": { "API model identifiers": [ "Qwen2.5-Coder-32B-Instruct" ], "Harness version": "1.0.0", "Harness major version": "1", "Reasoning effort": null, "Configuration tags": [ "Model: Qwen2.5-Coder-32B-Instruct", "Org: Qwen", "System: Attempts - 1", "Mini: 1.0.0" ], "Publisher checked": true, "Dataset instances": 500, "Run repetitions/sample size": null, "Warning": null, "Trajectories": "https://docent.transluce.org/dashboard/f5288cc2-48b7-4df9-8e30-072d96b58606", "Uncertainty": "No publisher confidence interval supplied" } } ] }, { "id": "epoch", "label": "Epoch Capabilities Index", "url": "https://epoch.ai/eci", "methodologyUrl": "https://epoch.ai/data/eci-documentation/methodology", "asOf": "2026-10-25", "retrievedAt": "2026-10-25T04:34:11Z", "description": "Epoch AI's fitted general capability index; 90% bootstrap confidence intervals and benchmark coverage are preserved.", "coverage": "270 models; 2,888 model-by-benchmark observations across 59 benchmarks in the downloaded fit.", "caveats": [ "Epoch aggregates the highest score for each benchmark across thinking effort and inference providers within the same model/release-date group.", "Mixed Epoch AI evaluations, benchmark-creator evaluations, and model-developer reports. Administrator types follow Epoch documentation; blank per-row references remain unknown.", "ECI is an abstract unbounded scale, not a percentage. Intervals overlap substantially; point-estimate order is not a statistically established ranking. Scores can change when Epoch refits the full dataset.", "Keep separate from AA, LiveBench, and SWE-bench: ECI already incorporates overlapping benchmark evidence and combines settings.", "Missing confidence intervals for GPT-5 and Claude 3.5 Sonnet are intentional: these are fixed calibration anchors.", "Model release dates are not evaluation dates. Evaluation counts are benchmark observations, not question counts or repeated trials.", "Attribution: Epoch AI · CC BY 4.0." ], "metrics": [ { "key": "eci", "label": "ECI", "unit": "index points" } ], "license": { "name": "CC BY 4.0", "url": "https://creativecommons.org/licenses/by/4.0/" }, "rawSha256": { "README.md": "43f5df3edacb9701b66626b13e78a52e2f823f0fa68db997e7fac40bacfa3e71", "epoch_capabilities_index/eci_scores.csv": "540796e67e6158f290edfb97a2633ca871edf9f1a89c78fc9d06cd20ca33cf64", "epoch_capabilities_index/processed_data_for_eci.csv": "7ca30714b01970a28fd06c40b1f0f87c9ab3deebddb5e1f2dc45950754776f43", "model_metadata.csv": "f6824f0db8ff4e2815d59479beefba105059c6d8131f158a4f87d7fd5ea7c84b", "benchmark_metadata.csv": "979c73b823a2ea9f37d33e495065c7952226f389be012a4c75902567ad4dc2a6" }, "rows": [ { "id": "epoch-9243bbfe2cfeea47", "model": "Claude Opus 5.5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 168.83 }, "details": { "90% CI": "165.25 to 173.95", "Model release date": "2026-09-22", "Evaluations": "21 model-by-benchmark observations", "Versions": "claude-opus-5-5_high; claude-opus-5-5_max; claude-opus-5-5_medium; claude-opus-5-5_unknown; claude-opus-5-5_xhigh", "Evaluator provenance": "Epoch AI: 11; Benchmark creator: 10; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; MirrorCode; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SimpleQA Verified; SpireClimb; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-1b3c7f8e637e1454", "model": "GPT-6 Astra", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 168.81 }, "details": { "90% CI": "165.45 to 174.51", "Model release date": "2026-09-03", "Evaluations": "24 model-by-benchmark observations", "Versions": "gpt-6-astra_high; gpt-6-astra_max; gpt-6-astra_none; gpt-6-astra_promax; gpt-6-astra_unknown; gpt-6-astra_xhigh", "Evaluator provenance": "Epoch AI: 11; Benchmark creator: 13; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Balrog; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; MirrorCode; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SimpleQA Verified; SpireClimb; WeirdML; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://balrogai.com/; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-ec9f1afddf925a20", "model": "GPT-6.1 Sol", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 168.37 }, "details": { "90% CI": "165.00 to 174.43", "Model release date": "2026-09-29", "Evaluations": "18 model-by-benchmark observations", "Versions": "gpt-6.1-sol_max; gpt-6.1-sol_medium; gpt-6.1-sol_unknown; gpt-6.1-sol_xhigh", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; SpireClimb; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-a5ee67b5769bd618", "model": "Claude Fable 5.1", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 166.49 }, "details": { "90% CI": "163.47 to 171.35", "Model release date": "2026-09-01", "Evaluations": "22 model-by-benchmark observations", "Versions": "claude-fable-5-1_high; claude-fable-5-1_max; claude-fable-5-1_medium; claude-fable-5-1_unknown; claude-fable-5-1_xhigh", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 12; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GSO-Bench; LMCA; MirrorCode; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SimpleQA Verified; SpireClimb; WeirdML; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-4ce5ae17197576b7", "model": "Gemini 4 Argon", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 166.45 }, "details": { "90% CI": "163.23 to 171.52", "Model release date": "2026-09-30", "Evaluations": "18 model-by-benchmark observations", "Versions": "gemini-4-argon_high; gemini-4-argon_low; gemini-4-argon_medium; gemini-4-argon_unknown", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Limited access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-21d97601528d7cff", "model": "Claude Sonnet 5.5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 166.18 }, "details": { "90% CI": "163.06 to 171.34", "Model release date": "2026-09-28", "Evaluations": "16 model-by-benchmark observations", "Versions": "claude-sonnet-5-5_max; claude-sonnet-5-5_xhigh", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; SpireClimb; WeirdML v3", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-cd8eedc764b60783", "model": "Claude Opus 5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 163.94 }, "details": { "90% CI": "161.12 to 168.24", "Model release date": "2026-07-24", "Evaluations": "23 model-by-benchmark observations", "Versions": "claude-opus-5; claude-opus-5_low; claude-opus-5_max; claude-opus-5_unknown; claude-opus-5_xhigh", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 14; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Balrog; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SimpleBench; SimpleQA Verified; WeirdML; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://balrogai.com/; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://osworld-v2.xlang.ai/; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-fe09781fb6df8780", "model": "GPT-6 Sol", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 163.73 }, "details": { "90% CI": "160.92 to 167.44", "Model release date": "2026-09-22", "Evaluations": "18 model-by-benchmark observations", "Versions": "gpt-6-sol_max; gpt-6-sol_unknown; gpt-6-sol_xhigh", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; SpireClimb; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-3456ef96946f535f", "model": "Claude Fable 5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 163.42 }, "details": { "90% CI": "160.53 to 168.18", "Model release date": "2026-06-09", "Evaluations": "25 model-by-benchmark observations", "Versions": "claude-fable-5; claude-fable-5_high; claude-fable-5_low; claude-fable-5_max; claude-fable-5_unknown; claude-fable-5_xhigh", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 15; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; MirrorCode; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; Remote Labor Index; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML; WeirdML v3", "Source references": "SimpleBench Leaderboard; https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-22e77de72b898d0e", "model": "GPT-5.5 Pro", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 162.91 }, "details": { "90% CI": "160.04 to 166.60", "Model release date": "2026-04-23", "Evaluations": "10 model-by-benchmark observations", "Versions": "gpt-5.5-pro-pre-release_xhigh; gpt-5.5-pro_high; gpt-5.5-pro_unknown; gpt-5.5-pro_xhigh", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; SimpleBench", "Source references": "https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://lmcouncil.ai/benchmarks" } }, { "id": "epoch-4c15583ff6e2f485", "model": "GPT-5.6 Sol", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 162.79 }, "details": { "90% CI": "160.15 to 167.06", "Model release date": "2026-07-09", "Evaluations": "26 model-by-benchmark observations", "Versions": "gpt-5.6-sol_high; gpt-5.6-sol_low; gpt-5.6-sol_max; gpt-5.6-sol_none; gpt-5.6-sol_promax; gpt-5.6-sol_proxhigh; gpt-5.6-sol_unknown; gpt-5.6-sol_xhigh", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 16; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Balrog; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; MirrorCode; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://balrogai.com/; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://osworld-v2.xlang.ai/; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-61171b554cfee30f", "model": "Muse Wonder", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 161.85 }, "details": { "90% CI": "159.39 to 165.44", "Model release date": "2026-10-12", "Evaluations": "10 model-by-benchmark observations", "Versions": "muse-wonder-1.0_max; muse-wonder-1.0_unknown; muse-wonder-1.0_xhigh", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; WeirdML v3", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-2719c688ad2de027", "model": "GPT-5.6 Terra", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 160.09 }, "details": { "90% CI": "157.30 to 163.31", "Model release date": "2026-07-09", "Evaluations": "19 model-by-benchmark observations", "Versions": "gpt-5.6-terra_high; gpt-5.6-terra_max; gpt-5.6-terra_medium; gpt-5.6-terra_none; gpt-5.6-terra_unknown; gpt-5.6-terra_xhigh", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 11; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Balrog; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "SimpleBench Leaderboard; https://arcprize.org/leaderboard; https://balrogai.com/; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-8f07378d04631b33", "model": "GPT-5.5", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 159.68 }, "details": { "90% CI": "157.33 to 163.67", "Model release date": "2026-04-23", "Evaluations": "29 model-by-benchmark observations", "Versions": "gpt-5.5-pre-release_xhigh; gpt-5.5_high; gpt-5.5_low; gpt-5.5_none; gpt-5.5_unknown; gpt-5.5_xhigh", "Evaluator provenance": "Epoch AI: 11; Benchmark creator: 18; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; CL-bench Life; Chess Puzzles; DTBench; DeepResearch Bench; EBR-bench; ExploitBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; MirrorCode; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; Remote Labor Index; SWE-Bench verified; SimpleBench; SimpleQA Verified; Surface Evolver Bench; Terminal Bench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://osworld-v2.xlang.ai/; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-bfdb7a2a68cfcaa4", "model": "GPT-5.4 Pro", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 159.57 }, "details": { "90% CI": "157.10 to 162.70", "Model release date": "2026-03-05", "Evaluations": "9 model-by-benchmark observations", "Versions": "gpt-5.4-pro-2026-03-05_none; gpt-5.4-pro-2026-03-05_unknown; gpt-5.4-pro-2026-03-05_xhigh", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; SimpleBench; SimpleQA Verified; WeirdML", "Source references": "SimpleBench Leaderboard; https://arcprize.org/leaderboard; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-011da203aeec2f36", "model": "Claude Haiku 5.5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 159.32 }, "details": { "90% CI": "156.19 to 162.47", "Model release date": "2026-10-07", "Evaluations": "18 model-by-benchmark observations", "Versions": "claude-haiku-5-5_max; claude-haiku-5-5_xhigh", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-ca919179bb3d7990", "model": "Claude Opus 4.8", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 159.01 }, "details": { "90% CI": "156.89 to 161.95", "Model release date": "2026-05-28", "Evaluations": "24 model-by-benchmark observations", "Versions": "claude-opus-4-8_high; claude-opus-4-8_max; claude-opus-4-8_none; claude-opus-4-8_unknown; claude-opus-4-8_xhigh", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 15; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; DeepResearch Bench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; Remote Labor Index; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://osworld-v2.xlang.ai/; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-947c93ec19c0e402", "model": "Gemini 3.7 Flash", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 157.97 }, "details": { "90% CI": "155.92 to 161.24", "Model release date": "2026-08-13", "Evaluations": "17 model-by-benchmark observations", "Versions": "gemini-3.7-flash_high; gemini-3.7-flash_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SimpleQA Verified", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-cacf6b25954a57c2", "model": "Kimi K3", "org": "Moonshot", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 157.86 }, "details": { "90% CI": "155.11 to 160.63", "Model release date": "2026-07-16", "Evaluations": "21 model-by-benchmark observations", "Versions": "kimi-k3_high; kimi-k3_low; kimi-k3_max; kimi-k3_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 13; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-1b6b70b55ddf774f", "model": "Gemini 3.8 Flash", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 157.44 }, "details": { "90% CI": "155.07 to 161.08", "Model release date": "2026-09-02", "Evaluations": "20 model-by-benchmark observations", "Versions": "gemini-3.8-flash_high; gemini-3.8-flash_medium; gemini-3.8-flash_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 12; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SimpleQA Verified; Surface Evolver Bench; WeirdML; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-ec19fd19ceb56069", "model": "Qwen 3.8 Max", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 157.4 }, "details": { "90% CI": "155.33 to 160.31", "Model release date": "2026-08-02", "Evaluations": "12 model-by-benchmark observations", "Versions": "qwen3.8-max_unknown; qwen3.8-max_xhigh", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-bc29172dffc618fc", "model": "GPT-5.4", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 157.31 }, "details": { "90% CI": "155.38 to 159.80", "Model release date": "2026-03-05", "Evaluations": "25 model-by-benchmark observations", "Versions": "gpt-5.4-2026-03-05_high; gpt-5.4-2026-03-05_low; gpt-5.4-2026-03-05_medium; gpt-5.4-2026-03-05_none; gpt-5.4-2026-03-05_unknown; gpt-5.4-2026-03-05_xhigh", "Evaluator provenance": "Epoch AI: 11; Benchmark creator: 14; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; CL-bench; CL-bench Life; Chess Puzzles; DTBench; DeepResearch Bench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; METR Time Horizons; MirrorCode; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SWE-Bench verified; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-e2adca9beef21cd3", "model": "Muse Spark 1.3", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 157.21 }, "details": { "90% CI": "155.26 to 159.82", "Model release date": "2026-09-02", "Evaluations": "11 model-by-benchmark observations", "Versions": "muse-spark-1.3_max; muse-spark-1.3_unknown; muse-spark-1.3_xhigh", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; WeirdML v3", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-a89ee82d1dfe8d5f", "model": "GPT-5.3 Codex", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 157.09 }, "details": { "90% CI": "153.56 to 161.38", "Model release date": "2026-02-05", "Evaluations": "4 model-by-benchmark observations", "Versions": "gpt-5.3-codex; gpt-5.3-codex_high", "Evaluator provenance": "Epoch AI: 1; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "METR Time Horizons; SWE-Bench verified; Terminal Bench; WeirdML", "Source references": "WeirdML Leaderboard; https://www.tbench.ai/leaderboard/terminal-bench/2.0" } }, { "id": "epoch-061c2e29e26808a0", "model": "Claude Sonnet 5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 156.88 }, "details": { "90% CI": "154.10 to 159.65", "Model release date": "2026-06-30", "Evaluations": "16 model-by-benchmark observations", "Versions": "claude-sonnet-5; claude-sonnet-5_high; claude-sonnet-5_max; claude-sonnet-5_medium; claude-sonnet-5_unknown; claude-sonnet-5_xhigh", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; GSO-Bench; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "SimpleBench Leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-b575622cfba6f284", "model": "Claude Opus 4.7", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 156.79 }, "details": { "90% CI": "154.91 to 159.19", "Model release date": "2026-04-16", "Evaluations": "25 model-by-benchmark observations", "Versions": "claude-opus-4-7; claude-opus-4-7_high; claude-opus-4-7_max; claude-opus-4-7_unknown; claude-opus-4-7_xhigh", "Evaluator provenance": "Epoch AI: 11; Benchmark creator: 14; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; ExploitBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; MirrorCode; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://osworld-v2.xlang.ai/; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-709ce67631842585", "model": "GPT-6 Luna", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 156.75 }, "details": { "90% CI": "154.32 to 159.43", "Model release date": "2026-09-22", "Evaluations": "17 model-by-benchmark observations", "Versions": "gpt-6-luna_max; gpt-6-luna_unknown; gpt-6-luna_xhigh", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; SpireClimb; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-4e2ddcfc52b369f9", "model": "Grok 4.6", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 156.57 }, "details": { "90% CI": "154.68 to 159.19", "Model release date": "2026-08-12", "Evaluations": "19 model-by-benchmark observations", "Versions": "grok-4.6_high; grok-4.6_medium; grok-4.6_unknown; grok-4.6_xhigh", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 10; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; SimpleQA Verified; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-95d3f179cfe80e1b", "model": "GPT-5.6 Luna", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 156.5 }, "details": { "90% CI": "154.19 to 159.26", "Model release date": "2026-07-09", "Evaluations": "20 model-by-benchmark observations", "Versions": "gpt-5.6-luna_high; gpt-5.6-luna_max; gpt-5.6-luna_medium; gpt-5.6-luna_none; gpt-5.6-luna_unknown; gpt-5.6-luna_xhigh", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 12; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Balrog; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML; WeirdML v3", "Source references": "SimpleBench Leaderboard; https://arcprize.org/leaderboard; https://balrogai.com/; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-bb65db55bb22cdbe", "model": "GPT-5.2 Pro", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 156.33 }, "details": { "90% CI": "153.89 to 159.98", "Model release date": "2025-12-11", "Evaluations": "6 model-by-benchmark observations", "Versions": "gpt-5.2-pro-2025-12-11; gpt-5.2-pro-2025-12-11_high; gpt-5.2-pro-2025-12-11_xhigh", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; SimpleBench", "Source references": "https://arcprize.org/leaderboard; https://lmcouncil.ai/benchmarks" } }, { "id": "epoch-a8bfe9eb2a1ce6c1", "model": "GLM-5.3", "org": "Z.ai (Zhipu AI)", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.99 }, "details": { "90% CI": "154.17 to 158.35", "Model release date": "2026-08-14", "Evaluations": "14 model-by-benchmark observations", "Versions": "glm-5.3_max; glm-5.3_unknown", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-45b344d42820d93b", "model": "DeepSeek V4.1 Flash", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.93 }, "details": { "90% CI": "153.68 to 158.59", "Model release date": "2026-09-09", "Evaluations": "17 model-by-benchmark observations", "Versions": "deepseek-v4.1-flash_high; deepseek-v4.1-flash_max; deepseek-v4.1-flash_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; Surface Evolver Bench; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-2902cfcf44bca341", "model": "Claude Opus 4.6", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.66 }, "details": { "90% CI": "153.82 to 158.07", "Model release date": "2026-02-05", "Evaluations": "27 model-by-benchmark observations", "Versions": "claude-opus-4-6; claude-opus-4-6_120K; claude-opus-4-6_high; claude-opus-4-6_max; claude-opus-4-6_unknown", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 16; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; CL-bench; CL-bench Life; Chess Puzzles; Cybench; DTBench; DeepResearch Bench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; METR Time Horizons; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "GSO Leaderboard; Opus 4.6 System Card; https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-45c9ea1cd8e20e3c", "model": "DeepSeek V4 Pro 0813", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.64 }, "details": { "90% CI": "153.74 to 158.04", "Model release date": "2026-08-13", "Evaluations": "15 model-by-benchmark observations", "Versions": "deepseek-v4-pro-0813_high; deepseek-v4-pro-0813_low; deepseek-v4-pro-0813_max; deepseek-v4-pro-0813_unknown", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-18a24eab73044b5a", "model": "Gemini 3.1 Pro", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.34 }, "details": { "90% CI": "153.32 to 158.13", "Model release date": "2026-02-19", "Evaluations": "28 model-by-benchmark observations", "Versions": "gemini-3.1-pro-preview; gemini-3.1-pro-preview-customtools; gemini-3.1-pro-preview_high; gemini-3.1-pro-preview_medium", "Evaluator provenance": "Epoch AI: 11; Benchmark creator: 17; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Balrog; CL-bench; CL-bench Life; Chess Puzzles; DTBench; DeepResearch Bench; EBR-bench; ExploitBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; GSO-Bench; LMCA; METR Time Horizons; MirrorCode; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "ARC Prize Leaderboard; Balrog Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-8ea0b93430d6d9bf", "model": "Muse Spark Open", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.11 }, "details": { "90% CI": "152.66 to 157.97", "Model release date": "2026-10-15", "Evaluations": "6 model-by-benchmark observations", "Versions": "Muse-Spark-Open-816B-A42B_unknown; Muse-Spark-Open-816B-A42B_xhigh", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 1; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "APEX-Agents; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; OTIS Mock AIME 2024-2025; SimpleQA Verified", "Source references": "Not individually reported" } }, { "id": "epoch-db3a00d1bf812384", "model": "Qwen3.8 Max (0902)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.08 }, "details": { "90% CI": "153.03 to 157.19", "Model release date": "2026-09-01", "Evaluations": "8 model-by-benchmark observations", "Versions": "qwen3.8-max-0902_xhigh", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 1; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Chess Puzzles; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; OTIS Mock AIME 2024-2025; SimpleQA Verified", "Source references": "https://www.frontierswe.com/" } }, { "id": "epoch-9f0e4feef3f69b3b", "model": "Muse Spark 1.2", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 155.05 }, "details": { "90% CI": "152.82 to 157.56", "Model release date": "2026-08-05", "Evaluations": "9 model-by-benchmark observations", "Versions": "muse-spark-1.2; muse-spark-1.2_xhigh", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; FrontierSWE; LMCA; ProofBench; SimpleBench; SimpleQA Verified; WeirdML", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-371dab1ad2a41cfe", "model": "Gemini 3.5 Flash", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 154.88 }, "details": { "90% CI": "152.91 to 157.41", "Model release date": "2026-05-19", "Evaluations": "18 model-by-benchmark observations", "Versions": "gemini-3.5-flash_high; gemini-3.5-flash_low; gemini-3.5-flash_medium; gemini-3.5-flash_unknown", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "SimpleBench Leaderboard; https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-b615c1e4921f7fea", "model": "DeepSeek V4 Flash 0731", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 154.76 }, "details": { "90% CI": "152.06 to 156.86", "Model release date": "2026-07-31", "Evaluations": "15 model-by-benchmark observations", "Versions": "deepseek-v4-flash-0731_max; deepseek-v4-flash-0731_unknown", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; SimpleQA Verified; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-c48adb0bec4f98ad", "model": "Gemini 3.6 Flash", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 154.53 }, "details": { "90% CI": "152.85 to 156.58", "Model release date": "2026-07-21", "Evaluations": "16 model-by-benchmark observations", "Versions": "gemini-3.6-flash_high; gemini-3.6-flash_minimal; gemini-3.6-flash_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-0848fce822d5ed97", "model": "Grok 4.7", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 154.37 }, "details": { "90% CI": "152.60 to 157.08", "Model release date": "2026-09-21", "Evaluations": "17 model-by-benchmark observations", "Versions": "grok-4.7_high; grok-4.7_unknown; grok-4.7_xhigh", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-eba10bb66138299a", "model": "Grok 4.5", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 154.07 }, "details": { "90% CI": "152.36 to 156.18", "Model release date": "2026-07-08", "Evaluations": "18 model-by-benchmark observations", "Versions": "grok-4.5_high; grok-4.5_medium; grok-4.5_unknown", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 11; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-d1fe05702b9f9f9c", "model": "Muse Spark 1.1", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 154.05 }, "details": { "90% CI": "151.73 to 156.18", "Model release date": "2026-07-09", "Evaluations": "7 model-by-benchmark observations", "Versions": "muse-spark-1.1; muse-spark-1.1_high; muse-spark-1.1_xhigh", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; LMCA; ProofBench; SimpleQA Verified; Surface Evolver Bench", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-96507293a0a4fe6a", "model": "Qwen3.7-Max", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 154.04 }, "details": { "90% CI": "152.30 to 156.50", "Model release date": "2026-05-19", "Evaluations": "13 model-by-benchmark observations", "Versions": "qwen3.7-max; qwen3.7-max_none", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Chess Puzzles; DTBench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleBench; SimpleQA Verified", "Source references": "SimpleBench Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-4dda9489c142ee68", "model": "GPT-5.2", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 153.66 }, "details": { "90% CI": "152.05 to 155.42", "Model release date": "2025-12-11", "Evaluations": "26 model-by-benchmark observations", "Versions": "gpt-5.2-2025-12-11_high; gpt-5.2-2025-12-11_low; gpt-5.2-2025-12-11_medium; gpt-5.2-2025-12-11_none; gpt-5.2-2025-12-11_unknown; gpt-5.2-2025-12-11_xhigh", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 16; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; CL-bench; Chess Puzzles; DTBench; DeepResearch Bench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GDPval; GPQA diamond; GSO-Bench; LMCA; METR Time Horizons; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; VPCT; WeirdML; WeirdML v3", "Source references": "ARC Prize Leaderboard; SimpleBench Leaderboard; VPCT leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-e8f0d28d3c21c3b3", "model": "GLM-5.3-Flash", "org": "Z.ai (Zhipu AI)", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 153.08 }, "details": { "90% CI": "150.76 to 155.53", "Model release date": "2026-08-20", "Evaluations": "13 model-by-benchmark observations", "Versions": "glm-5.3-flash_max; glm-5.3-flash_unknown", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; GPQA diamond; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Surface Evolver Bench", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-2f96e3cdfc593a90", "model": "Gemini 3 Pro", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 153.07 }, "details": { "90% CI": "151.06 to 155.42", "Model release date": "2025-11-18", "Evaluations": "20 model-by-benchmark observations", "Versions": "gemini-3-pro-preview; gemini-3-pro-preview_low", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 15; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Balrog; CL-bench; DeepResearch Bench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GDPval; GPQA diamond; GSO-Bench; GeoBench; METR Time Horizons; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SWE-Bench verified; SimpleBench; Terminal Bench; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; GeoBench leaderboard; VPCT leaderboard; https://balrogai.com/; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-11bf1657f662888b", "model": "Claude Sonnet 4.6", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 152.79 }, "details": { "90% CI": "150.20 to 155.06", "Model release date": "2026-02-17", "Evaluations": "21 model-by-benchmark observations", "Versions": "claude-sonnet-4-6; claude-sonnet-4-6_16K; claude-sonnet-4-6_high; claude-sonnet-4-6_max; claude-sonnet-4-6_medium; claude-sonnet-4-6_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 13; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; DeepResearch Bench; ExploitBench; FrontierCode; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OSWorld; OSWorld 2.0; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "OS World Website; WeirdML Leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://osworld-v2.xlang.ai/; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-1df02a89dd910e15", "model": "Grok 4.20", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 152.11 }, "details": { "90% CI": "149.57 to 154.54", "Model release date": "2026-02-17", "Evaluations": "15 model-by-benchmark observations", "Versions": "grok-4-20; grok-4.20-0309-reasoning", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; CL-bench; CL-bench Life; Chess Puzzles; DTBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-61dc4a612a111e57", "model": "Muse Spark", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 152.03 }, "details": { "90% CI": "149.79 to 154.49", "Model release date": "2026-04-08", "Evaluations": "5 model-by-benchmark observations", "Versions": "muse-spark", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 1; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; OTIS Mock AIME 2024-2025; ProofBench", "Source references": "https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-8efd4aa5f78ad9b5", "model": "GLM-5.2", "org": "Z.ai (Zhipu AI)", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 151.94 }, "details": { "90% CI": "149.99 to 154.10", "Model release date": "2026-06-16", "Evaluations": "20 model-by-benchmark observations", "Versions": "glm-5.2_high; glm-5.2_max; glm-5.2_medium; glm-5.2_none; glm-5.2_unknown", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 11; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; EBR-bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; PostTrainBench; ProofBench; SWE-Bench verified; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-d9baf7ed767e9c96", "model": "Gemini 3 Flash", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 151.88 }, "details": { "90% CI": "150.44 to 153.64", "Model release date": "2025-12-17", "Evaluations": "21 model-by-benchmark observations", "Versions": "gemini-3-flash-preview; gemini-3-flash-preview_high; gemini-3-flash-preview_low; gemini-3-flash-preview_minimal", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 13; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Balrog; Chess Puzzles; DTBench; DeepResearch Bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; GSO-Bench; GeoBench; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; GeoBench leaderboard; VPCT leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-ee75ceb70d272aff", "model": "Qwen 3.8 27B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 151.69 }, "details": { "90% CI": "149.75 to 153.72", "Model release date": "2026-08-14", "Evaluations": "7 model-by-benchmark observations", "Versions": "qwen3.8-27b; qwen3.8-27b_xhigh", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; DTBench; LMCA; ProofBench; Surface Evolver Bench", "Source references": "https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-2a36290594fff2da", "model": "Kimi K2.6", "org": "Moonshot", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 150.94 }, "details": { "90% CI": "149.04 to 152.72", "Model release date": "2026-04-20", "Evaluations": "16 model-by-benchmark observations", "Versions": "kimi-k2.6; kimi-k2.6_none", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Chess Puzzles; DTBench; EBR-bench; ExploitBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleQA Verified; WeirdML", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://osworld-v2.xlang.ai/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-b4421acff2f5aa4d", "model": "GPT-5 Pro", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 150.59 }, "details": { "90% CI": "148.94 to 154.18", "Model release date": "2025-10-07", "Evaluations": "7 model-by-benchmark observations", "Versions": "gpt-5-pro-2025-10-06_high; gpt-5-pro-2025-10-06_unknown", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; SimpleBench; WeirdML", "Source references": "ARC Prize Leaderboard; SimpleBench Leaderboard; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-60d3c69f0791526d", "model": "Inkling-Small", "org": "Thinking Machines", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 150.32 }, "details": { "90% CI": "147.23 to 152.10", "Model release date": "2026-07-15", "Evaluations": "9 model-by-benchmark observations", "Versions": "inkling-small; inkling-small_xhigh", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified", "Source references": "https://arcprize.org/leaderboard; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-7d099cbc635f5e7c", "model": "Claude Opus 4.5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 150.11 }, "details": { "90% CI": "147.81 to 153.01", "Model release date": "2025-11-24", "Evaluations": "30 model-by-benchmark observations", "Versions": "claude-opus-4-5-20251101; claude-opus-4-5-20251101_128K; claude-opus-4-5-20251101_16K; claude-opus-4-5-20251101_32K; claude-opus-4-5-20251101_48K; claude-opus-4-5-20251101_59K; claude-opus-4-5-20251101_64K; claude-opus-4-5-20251101_high; claude-opus-4-5-20251101_unknown", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 19; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Balrog; CL-bench; Chess Puzzles; Cybench; DTBench; DeepResearch Bench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GDPval; GPQA diamond; GSO-Bench; GeoBench; LMCA; METR Time Horizons; Mystery Game Puzzles; OSWorld; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; VPCT; WeirdML; WeirdML v3", "Source references": "ARC Prize Leaderboard; Anthropic annoucement; Balrog Leaderboard; GSO Leaderboard; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; Opus 4.5 System Card; SimpleBench Leaderboard; Tweet from VPCT creator; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-ea53fba6906cd294", "model": "Kimi K2.7 Code", "org": "Moonshot", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 150.11 }, "details": { "90% CI": "148.17 to 151.82", "Model release date": "2026-06-12", "Evaluations": "11 model-by-benchmark observations", "Versions": "kimi-k2.7-code", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; Chess Puzzles; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; OTIS Mock AIME 2024-2025; SimpleBench; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "https://cognition.com/frontiercode; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-8f85116e4332ff52", "model": "Mistral Large 4", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 150.03 }, "details": { "90% CI": "146.94 to 152.74", "Model release date": "2026-10-06", "Evaluations": "14 model-by-benchmark observations", "Versions": "mistral-large-4-0_high; mistral-large-4-0_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; Furniture Assembly; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; Surface Evolver Bench; WeirdML v3", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-ff2529f856f4d2ed", "model": "GPT-5", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 150.0 }, "details": { "90% CI": "Fixed calibration anchor; interval not published", "Model release date": "2025-08-07", "Evaluations": "29 model-by-benchmark observations", "Versions": "gpt-5-2025-08-07_high; gpt-5-2025-08-07_low; gpt-5-2025-08-07_medium; gpt-5-2025-08-07_minimal; gpt-5-2025-08-07_unknown", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 19; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Balrog; Chess Puzzles; DTBench; DeepResearch Bench; EBR-bench; Fiction.LiveBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GDPval; GPQA diamond; GSO-Bench; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; METR Time Horizons; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; VPCT; WeirdML", "Source references": "Fiction.live leaderboard; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://arcprize.org/leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench; lechmazur/writing Github repository" } }, { "id": "epoch-539e801876c70558", "model": "GLM-5.1", "org": "Z.ai (Zhipu AI)", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 149.82 }, "details": { "90% CI": "147.66 to 151.89", "Model release date": "2026-04-07", "Evaluations": "12 model-by-benchmark observations", "Versions": "glm-5.1", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; Chess Puzzles; ExploitBench; FrontierMath-Tier-4-2025-07-01-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleBench; SimpleQA Verified; WeirdML", "Source references": "SimpleBench Leaderboard; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-44700474f3c5f932", "model": "GPT-5.1", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 149.76 }, "details": { "90% CI": "148.36 to 151.74", "Model release date": "2025-11-13", "Evaluations": "20 model-by-benchmark observations", "Versions": "gpt-5.1-2025-11-13_high; gpt-5.1-2025-11-13_low; gpt-5.1-2025-11-13_medium; gpt-5.1-2025-11-13_none; gpt-5.1-2025-11-13_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 12; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; CL-bench; CL-bench Life; Chess Puzzles; DTBench; DeepResearch Bench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; GSO-Bench; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; VPCT; WeirdML", "Source references": "Terminal-Bench v2 Leaderboard; VPCT leaderboard; https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://gso-bench.github.io/index.html; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks" } }, { "id": "epoch-252b8b957738f60f", "model": "Qwen 3.6 Max (Preview)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 149.41 }, "details": { "90% CI": "147.60 to 153.15", "Model release date": "2026-04-20", "Evaluations": "10 model-by-benchmark observations", "Versions": "qwen3.6-max-preview; qwen3.6-max-preview_none", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "DTBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; SimpleQA Verified", "Source references": "SimpleBench Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-9db5201cc1f8dc04", "model": "DeepSeek-V4-Pro", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 149.13 }, "details": { "90% CI": "147.63 to 150.87", "Model release date": "2026-04-24", "Evaluations": "15 model-by-benchmark observations", "Versions": "deepseek-v4-pro_high; deepseek-v4-pro_max; deepseek-v4-pro_none", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "CL-bench Life; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-9bee3638f39d8e00", "model": "Grok 4.3 Beta", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 148.81 }, "details": { "90% CI": "147.21 to 150.34", "Model release date": "2026-04-17", "Evaluations": "10 model-by-benchmark observations", "Versions": "grok-4.3_high; grok-4.3_unknown", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Hosted access (no API)", "Benchmarks": "Chess Puzzles; DTBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-4e917cf200726bf5", "model": "GPT-5.4 Mini", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 148.7 }, "details": { "90% CI": "146.87 to 150.47", "Model release date": "2026-03-17", "Evaluations": "15 model-by-benchmark observations", "Versions": "gpt-5.4-mini-2026-03-17_high; gpt-5.4-mini-2026-03-17_low; gpt-5.4-mini-2026-03-17_medium; gpt-5.4-mini-2026-03-17_none; gpt-5.4-mini-2026-03-17_unknown; gpt-5.4-mini-2026-03-17_xhigh", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; DeepResearch Bench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-24d5400a33446098", "model": "Inkling", "org": "Thinking Machines", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 148.41 }, "details": { "90% CI": "145.96 to 150.21", "Model release date": "2026-07-15", "Evaluations": "17 model-by-benchmark observations", "Versions": "Inkling; Inkling_high; Inkling_xhigh", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 11; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierCode; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; FrontierSWE; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; SimpleQA Verified; WeirdML; WeirdML v3", "Source references": "https://arcprize.org/leaderboard; https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; https://www.frontierswe.com/; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-97a650582ba3a3f9", "model": "Kimi K2.5", "org": "Moonshot", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 148.07 }, "details": { "90% CI": "146.55 to 149.52", "Model release date": "2026-01-27", "Evaluations": "15 model-by-benchmark observations", "Versions": "fireworks/kimi-k2p5; kimi-k2.5", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; CL-bench; CL-bench Life; Fiction.LiveBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; OSWorld; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "Fiction.live leaderboard; OS World Website; SimpleBench Leaderboard; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0" } }, { "id": "epoch-109f0072c1fda3cc", "model": "o3-pro", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 147.33 }, "details": { "90% CI": "145.56 to 150.10", "Model release date": "2025-06-10", "Evaluations": "8 model-by-benchmark observations", "Versions": "o3-pro-2025-06-10_high; o3-pro-2025-06-10_medium", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; DTBench; Fiction.LiveBench; LMCA; Lech Mazur Writing; WeirdML", "Source references": "Fiction.live leaderboard; https://aider.chat/docs/leaderboards/; https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-157f8a956dfba17b", "model": "Qwen 3.6 Plus", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 147.3 }, "details": { "90% CI": "145.07 to 149.26", "Model release date": "2026-03-31", "Evaluations": "11 model-by-benchmark observations", "Versions": "qwen3.6-plus; qwen3.6-plus_none", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "CL-bench; Chess Puzzles; DTBench; FrontierMath-Tier-4-2025-07-01-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleQA Verified", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-d28245944782f5b6", "model": "Qwen3.7-Plus", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 147.0 }, "details": { "90% CI": "145.25 to 148.82", "Model release date": "2026-06-02", "Evaluations": "8 model-by-benchmark observations", "Versions": "qwen3.7-plus; qwen3.7-plus_none", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; FrontierCode; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://osworld-v2.xlang.ai/" } }, { "id": "epoch-ac72b9b986af3932", "model": "MiniMax-M3", "org": "MiniMax", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.87 }, "details": { "90% CI": "142.34 to 150.17", "Model release date": "2026-06-01", "Evaluations": "12 model-by-benchmark observations", "Versions": "MiniMax-M3; MiniMax-M3_none", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "APEX-Agents; Chess Puzzles; DTBench; FrontierCode; GPQA diamond; LMCA; Mystery Game Puzzles; OSWorld 2.0; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; Surface Evolver Bench", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://lmcouncil.ai/benchmarks; https://osworld-v2.xlang.ai/; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-d9a93d9b6596075b", "model": "Qwen3.5 397B-A17B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.8 }, "details": { "90% CI": "145.24 to 148.86", "Model release date": "2026-02-13", "Evaluations": "7 model-by-benchmark observations", "Versions": "qwen3.5-397b-a17b; qwen3.5-397b-a17b_none", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; DTBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-e856df356cc92fba", "model": "Claude Sonnet 4.5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.74 }, "details": { "90% CI": "145.03 to 148.31", "Model release date": "2025-09-29", "Evaluations": "26 model-by-benchmark observations", "Versions": "claude-sonnet-4-5-20250929; claude-sonnet-4-5-20250929_12K; claude-sonnet-4-5-20250929_16K; claude-sonnet-4-5-20250929_2K; claude-sonnet-4-5-20250929_32K; claude-sonnet-4-5-20250929_48K; claude-sonnet-4-5-20250929_59K; claude-sonnet-4-5-20250929_unknown", "Evaluator provenance": "Epoch AI: 10; Benchmark creator: 15; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; Cybench; DTBench; DeepResearch Bench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GDPval; GPQA diamond; GSO-Bench; LMCA; MATH level 5; METR Time Horizons; Mystery Game Puzzles; OSWorld; OTIS Mock AIME 2024-2025; ProofBench; Remote Labor Index; SWE-Bench verified; SimpleBench; SimpleQA Verified; Terminal Bench; VPCT; WeirdML", "Source references": "DeepResearchBench Leaderboard; GSO Leaderboard; METR - Measuring AI Ability to Complete Long Tasks; OS World Website; Opus 4.5 System Card; SimpleBench Leaderboard; VPCT leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-de2d91dc0a258041", "model": "o3", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.73 }, "details": { "90% CI": "144.83 to 148.52", "Model release date": "2025-04-16", "Evaluations": "27 model-by-benchmark observations", "Versions": "o3-2025-04-16_high; o3-2025-04-16_low; o3-2025-04-16_medium; o3-2025-04-16_unknown", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 18; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; CL-bench; CadEval; Chess Puzzles; DTBench; DeepResearch Bench; Fiction.LiveBench; FrontierMath-Tier-4-2025-07-01-Private; FrontierMath-Tiers-1-3-v2-Private; GDPval; GPQA diamond; GSO-Bench; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; METR Time Horizons; Mystery Game Puzzles; OSWorld; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; SimpleQA Verified; VPCT; WeirdML", "Source references": "Aider LLM Leaderboards; CadEval Dashboard; Fiction.live leaderboard; GSO Leaderboard; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; OS World Website; SimpleBench Leaderboard; VPCT leaderboard; https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-32a6b67da07170b1", "model": "MiniMax-M2.5", "org": "MiniMax", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.59 }, "details": { "90% CI": "142.21 to 147.90", "Model release date": "2026-02-12", "Evaluations": "6 model-by-benchmark observations", "Versions": "MiniMax-M2.5", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; CL-bench; CL-bench Life; ProofBench; Terminal Bench", "Source references": "https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-7bbfd3233499a3ac", "model": "Grok 4", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.24 }, "details": { "90% CI": "144.36 to 148.02", "Model release date": "2025-07-09", "Evaluations": "18 model-by-benchmark observations", "Versions": "grok-4-0709; grok-4-0709_high", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 13; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Balrog; Cybench; DeepResearch Bench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GDPval; GPQA diamond; GeoBench; Lech Mazur Writing; METR Time Horizons; OTIS Mock AIME 2024-2025; SimpleBench; Terminal Bench; WeirdML", "Source references": "Aider LLM Leaderboards; Fiction.live leaderboard; GeoBench leaderboard; Grok 4 Model Card; METR - Measuring AI Ability to Complete Long Tasks; Terminal-Bench v2 Leaderboard; https://balrogai.com/; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks; lechmazur/writing Github repository" } }, { "id": "epoch-817a1ed13eb9b4be", "model": "Qwen3.6 27B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.23 }, "details": { "90% CI": "144.13 to 147.65", "Model release date": "2026-04-22", "Evaluations": "6 model-by-benchmark observations", "Versions": "qwen3.6-27b; qwen3.6-27b_none", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-a99008f0e5756a50", "model": "Qwen 3.5 Plus (hosted 397B-A17B)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.17 }, "details": { "90% CI": "143.48 to 147.89", "Model release date": "2026-02-16", "Evaluations": "11 model-by-benchmark observations", "Versions": "qwen3.5-plus; qwen3.5-plus_none", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "CL-bench; CL-bench Life; Chess Puzzles; DTBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleQA Verified", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-25ff2f3e5e17a3c4", "model": "DeepSeek-V4-Flash", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.11 }, "details": { "90% CI": "143.82 to 147.99", "Model release date": "2026-04-24", "Evaluations": "4 model-by-benchmark observations", "Versions": "deepseek-v4-flash_max; deepseek-v4-flash_unknown", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; LMCA; SimpleBench; WeirdML", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://lmcouncil.ai/benchmarks" } }, { "id": "epoch-28e8e5323b06e68c", "model": "DeepSeek-V3.2", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.09 }, "details": { "90% CI": "144.27 to 147.38", "Model release date": "2025-12-01", "Evaluations": "14 model-by-benchmark observations", "Versions": "deepseek-chat; deepseek-reasoner; deepseek/deepseek-v3.2; fireworks/deepseek-v3p2", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 10; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Aider polyglot; CL-bench; CL-bench Life; DTBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; ProofBench; Terminal Bench", "Source references": "https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-68a79c562fa5737d", "model": "GPT-5.4 Nano", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 146.03 }, "details": { "90% CI": "143.63 to 148.15", "Model release date": "2026-03-17", "Evaluations": "13 model-by-benchmark observations", "Versions": "gpt-5.4-nano-2026-03-17_high; gpt-5.4-nano-2026-03-17_none; gpt-5.4-nano-2026-03-17_xhigh", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-0e2484a13fd6b4c6", "model": "Nemotron 3 Ultra", "org": "Nvidia", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 145.97 }, "details": { "90% CI": "143.68 to 148.22", "Model release date": "2026-06-04", "Evaluations": "9 model-by-benchmark observations", "Versions": "nemotron-3-ultra", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; DTBench; FrontierCode; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; WeirdML", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-d22724b7a2eec23a", "model": "Kimi K2 Thinking", "org": "Moonshot", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 145.82 }, "details": { "90% CI": "143.22 to 147.51", "Model release date": "2025-11-06", "Evaluations": "8 model-by-benchmark observations", "Versions": "kimi-k2-thinking; kimi-k2-thinking-turbo; moonshotai/Kimi-K2-Thinking", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "CL-bench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; METR Time Horizons; OTIS Mock AIME 2024-2025; Terminal Bench; WeirdML", "Source references": "METR - Measuring AI Ability to Complete Long Tasks; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0" } }, { "id": "epoch-4a3c8d12f0d5d485", "model": "GLM-5", "org": "Z.ai (Zhipu AI)", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 145.8 }, "details": { "90% CI": "143.77 to 147.67", "Model release date": "2026-02-11", "Evaluations": "11 model-by-benchmark observations", "Versions": "glm-5", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; CL-bench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; Terminal Bench; WeirdML", "Source references": "SimpleBench Leaderboard; WeirdML Leaderboard; https://www.tbench.ai/leaderboard/terminal-bench/2.0" } }, { "id": "epoch-ea0d72ff356988b6", "model": "MiniMax-M2.7", "org": "MiniMax", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 145.78 }, "details": { "90% CI": "138.37 to 148.18", "Model release date": "2026-03-18", "Evaluations": "4 model-by-benchmark observations", "Versions": "MiniMax-M2.7", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "ExploitBench; ProofBench; Terminal Bench; WeirdML", "Source references": "https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-9056e3300ca806c0", "model": "o4-mini", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 145.46 }, "details": { "90% CI": "142.90 to 147.21", "Model release date": "2025-04-16", "Evaluations": "23 model-by-benchmark observations", "Versions": "o4-mini-2025-04-16_high; o4-mini-2025-04-16_low; o4-mini-2025-04-16_medium", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 15; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; CadEval; Chess Puzzles; DTBench; Fiction.LiveBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GDPval; GPQA diamond; GSO-Bench; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; METR Time Horizons; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleBench; SimpleQA Verified; VPCT; WeirdML", "Source references": "CadEval Dashboard; Fiction.live leaderboard; GSO Leaderboard; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-3c3ede7f1533fc78", "model": "GPT-5 mini", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 145.34 }, "details": { "90% CI": "143.33 to 147.11", "Model release date": "2025-08-07", "Evaluations": "19 model-by-benchmark observations", "Versions": "gpt-5-mini-2025-08-07_high; gpt-5-mini-2025-08-07_low; gpt-5-mini-2025-08-07_medium; gpt-5-mini-2025-08-07_minimal; gpt-5-mini-2025-08-07_unknown", "Evaluator provenance": "Epoch AI: 9; Benchmark creator: 10; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; Fiction.LiveBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Lech Mazur Writing; MATH level 5; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SWE-Bench verified; SimpleQA Verified; Terminal Bench; VPCT; WeirdML", "Source references": "Fiction.live leaderboard; VPCT leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench; lechmazur/writing Github repository" } }, { "id": "epoch-a201562c7d7940b6", "model": "Gemini 2.5 Pro (Jun 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 145.13 }, "details": { "90% CI": "143.45 to 146.69", "Model release date": "2025-06-05", "Evaluations": "21 model-by-benchmark observations", "Versions": "gemini-2.5-pro; gemini-2.5-pro-preview-06-05; gemini-2.5-pro_16K; gemini-2.5-pro_32K", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 16; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; DTBench; DeepResearch Bench; Fiction.LiveBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GDPval; GPQA diamond; GSO-Bench; LMCA; Lech Mazur Writing; METR Time Horizons; OTIS Mock AIME 2024-2025; Remote Labor Index; SWE-Bench verified; SimpleBench; Terminal Bench; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; DeepResearchBench Leaderboard; Fiction.live leaderboard; METR - Measuring AI Ability to Complete Long Tasks; VPCT leaderboard; WeirdML Leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://gso-bench.github.io/index.html; https://lmcouncil.ai/benchmarks; https://www.tbench.ai/leaderboard/terminal-bench/2.0; lechmazur/writing Github repository" } }, { "id": "epoch-fcde6b7025335af9", "model": "DeepSeek-V3.2-Exp", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 144.91 }, "details": { "90% CI": "142.45 to 146.81", "Model release date": "2025-09-29", "Evaluations": "8 model-by-benchmark observations", "Versions": "DeepSeek-V3.2-Exp; DeepSeek-V3.2-Exp_high; DeepSeek-V3.2-Exp_thinking", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Aider polyglot; CL-bench; CL-bench Life; DTBench; Fiction.LiveBench; LMCA; The Agent Company; WeirdML", "Source references": "Aider LLM Leaderboards; Fiction.live leaderboard; TheAgentCompany leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-aa9268f76b4a438d", "model": "Gemini 3.5 Flash-Lite", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 144.91 }, "details": { "90% CI": "142.41 to 146.26", "Model release date": "2026-07-21", "Evaluations": "13 model-by-benchmark observations", "Versions": "gemini-3.5-flash-lite; gemini-3.5-flash-lite_high; gemini-3.5-flash-lite_low; gemini-3.5-flash-lite_minimal", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "APEX-Agents; ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; WeirdML", "Source references": "https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-8e58bd1600019abd", "model": "Gemini 3.1 Flash-Lite", "org": "Google", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 144.26 }, "details": { "90% CI": "142.31 to 145.73", "Model release date": "2026-03-03", "Evaluations": "7 model-by-benchmark observations", "Versions": "gemini-3.1-flash-lite; gemini-3.1-flash-lite_high; gemini-3.1-flash-lite_low; gemini-3.1-flash-lite_minimal", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; DeepResearch Bench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "WeirdML Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb" } }, { "id": "epoch-aa5f70bd6142a9a9", "model": "Grok 4 Fast", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 144.09 }, "details": { "90% CI": "141.61 to 146.30", "Model release date": "2025-09-19", "Evaluations": "6 model-by-benchmark observations", "Versions": "grok-4-fast", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 5; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Cybench; DTBench; Fiction.LiveBench; WeirdML", "Source references": "Fiction.live leaderboard; Grok 4 Fast Model Card; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-eab5646fbfe336dd", "model": "Claude Opus 4.1", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 144.07 }, "details": { "90% CI": "141.75 to 146.55", "Model release date": "2025-08-05", "Evaluations": "18 model-by-benchmark observations", "Versions": "claude-opus-4-1-20250805; claude-opus-4-1-20250805_16K; claude-opus-4-1-20250805_24K; claude-opus-4-1-20250805_27K; claude-opus-4-1-20250805_32K; claude-opus-4-1-20250805_unknown", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 10; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Cybench; DTBench; DeepResearch Bench; EBR-bench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GDPval; GPQA diamond; LMCA; Lech Mazur Writing; METR Time Horizons; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; Terminal Bench; VPCT; WeirdML", "Source references": "Cybench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; Terminal-Bench v2 Leaderboard; VPCT leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-e4c52f2590a61975", "model": "Qwen3.7 Flash", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 144.05 }, "details": { "90% CI": "141.89 to 145.92", "Model release date": "2026-07-27", "Evaluations": "4 model-by-benchmark observations", "Versions": "qwen3.7-flash; qwen3.7-flash_none", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 0; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; Mystery Game Puzzles; OTIS Mock AIME 2024-2025", "Source references": "Not individually reported" } }, { "id": "epoch-10d7f9ee30b6bf3e", "model": "Gemini 2.5 Pro (Mar 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 143.96 }, "details": { "90% CI": "141.74 to 146.84", "Model release date": "2025-03-31", "Evaluations": "11 model-by-benchmark observations", "Versions": "gemini-2.5-pro-exp-03-25; gemini-2.5-pro-preview-03-25", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; Aider polyglot; Balrog; CadEval; Fiction.LiveBench; GPQA diamond; GeoBench; Lech Mazur Writing; MATH level 5; SimpleBench; VPCT", "Source references": "ARC Prize Leaderboard; CadEval Dashboard; Fiction.live leaderboard; GeoBench leaderboard; SimpleBench Leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://balrogai.com/; lechmazur/writing Github repository" } }, { "id": "epoch-a8384a23218e0393", "model": "Qwen3-235B-A22B-Thinking (Jul 2025)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 143.76 }, "details": { "90% CI": "141.71 to 145.30", "Model release date": "2025-07-25", "Evaluations": "11 model-by-benchmark observations", "Versions": "Qwen/Qwen3-235B-A22B-Thinking-2507; Qwen3-235B-A22B-Thinking-2507; chutes/Qwen3-235B-A22B-Thinking-2507", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; LMCA; Lech Mazur Writing; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleQA Verified; WeirdML", "Source references": "Fiction.live leaderboard; WeirdML Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; lechmazur/writing Github repository" } }, { "id": "epoch-82094e5584650818", "model": "Qwen 3.6 35B-A3B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 143.73 }, "details": { "90% CI": "141.05 to 145.80", "Model release date": "2026-04-14", "Evaluations": "9 model-by-benchmark observations", "Versions": "qwen3.6-35b-a3b; qwen3.6-35b-a3b_none", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; Surface Evolver Bench; Terminal Bench; WeirdML", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-9f17c730ead75e11", "model": "GLM-4.7", "org": "Z.ai (Zhipu AI)", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 143.59 }, "details": { "90% CI": "141.38 to 145.85", "Model release date": "2025-12-22", "Evaluations": "11 model-by-benchmark observations", "Versions": "glm-4.7; zai-org/GLM-4.7; zai-org/glm-4.7", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "CL-bench; CL-bench Life; Chess Puzzles; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; OTIS Mock AIME 2024-2025; ProofBench; SimpleBench; SimpleQA Verified; Terminal Bench", "Source references": "SimpleBench Leaderboard; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-7e07b64198f3236f", "model": "Qwen 3.5 Flash (hosted 35B-A3B)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 143.52 }, "details": { "90% CI": "141.48 to 145.04", "Model release date": "2026-02-25", "Evaluations": "9 model-by-benchmark observations", "Versions": "qwen3.5-flash; qwen3.5-flash_none", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "Chess Puzzles; DTBench; FrontierMath-Tier-4-2025-07-01-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleQA Verified", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-e82dbaef8d8c3cd7", "model": "Gemma 4 31B IT", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.99 }, "details": { "90% CI": "140.34 to 145.75", "Model release date": "2026-04-02", "Evaluations": "7 model-by-benchmark observations", "Versions": "gemma-4-31b-it; gemma-4-31b-it_minimal", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; SimpleQA Verified; Surface Evolver Bench; WeirdML", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-0670e890b1dcde03", "model": "Gemini 2.5 Flash (Sep 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.9 }, "details": { "90% CI": "139.12 to 146.17", "Model release date": "2025-09-25", "Evaluations": "4 model-by-benchmark observations", "Versions": "gemini-2.5-flash-preview-09-2025; gemini-2.5-flash-preview-09-2025_16k", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Terminal Bench; The Agent Company; VPCT; WeirdML", "Source references": "Terminal-Bench v2 Leaderboard; TheAgentCompany leaderboard; VPCT leaderboard; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-4a1d04970ebf8260", "model": "Qwen 3.6 Flash", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.89 }, "details": { "90% CI": "140.23 to 144.45", "Model release date": "2026-04-27", "Evaluations": "10 model-by-benchmark observations", "Versions": "qwen3.6-flash", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "Chess Puzzles; DTBench; FrontierMath-Tier-4-2025-07-01-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleBench; SimpleQA Verified", "Source references": "SimpleBench Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-471bcce32a67cdda", "model": "Claude Opus 4", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.52 }, "details": { "90% CI": "140.29 to 144.30", "Model release date": "2025-05-22", "Evaluations": "21 model-by-benchmark observations", "Versions": "claude-opus-4-20250514; claude-opus-4-20250514_12K; claude-opus-4-20250514_16K; claude-opus-4-20250514_27K; claude-opus-4-20250514_2K; claude-opus-4-20250514_32K; claude-opus-4-20250514_unknown", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 14; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Cybench; DTBench; DeepResearch Bench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; GSO-Bench; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; METR Time Horizons; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; VPCT; WeirdML", "Source references": "Cybench leaderboard; Fiction.live leaderboard; GSO Leaderboard; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-9a321acccb2fa475", "model": "Qwen3.5-35B-A3B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.49 }, "details": { "90% CI": "140.42 to 144.98", "Model release date": "2026-02-24", "Evaluations": "4 model-by-benchmark observations", "Versions": "qwen3.5-35b-a3b; qwen3.5-35b-a3b_none", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-f95116fec4f5ec7a", "model": "GPT-5.5 Instant", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.43 }, "details": { "90% CI": "139.73 to 145.28", "Model release date": "2026-05-05", "Evaluations": "4 model-by-benchmark observations", "Versions": "gpt-5.5-instant", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 0; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; OTIS Mock AIME 2024-2025", "Source references": "Not individually reported" } }, { "id": "epoch-c5c2fd2366e39d66", "model": "Qwen3-Max", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.33 }, "details": { "90% CI": "140.11 to 145.21", "Model release date": "2025-09-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "qwen3-max-2025-09-23", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "CL-bench; DTBench; Fiction.LiveBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; MATH level 5; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleQA Verified", "Source references": "Fiction.live leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-aae9e84e04d199a6", "model": "Claude Haiku 4.5", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.15 }, "details": { "90% CI": "139.40 to 144.06", "Model release date": "2025-10-15", "Evaluations": "16 model-by-benchmark observations", "Versions": "claude-haiku-4-5-20251001; claude-haiku-4-5-20251001_32K; claude-haiku-4-5-20251001_59K; claude-haiku-4-5-20251001_low; claude-haiku-4-5-20251001_unknown", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Balrog; Chess Puzzles; DTBench; DeepResearch Bench; ExploitBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; LMCA; MATH level 5; OTIS Mock AIME 2024-2025; SimpleQA Verified; Terminal Bench; WeirdML", "Source references": "Balrog Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0" } }, { "id": "epoch-ef9fbc377c0b0667", "model": "Gemini 2.5 Pro (May 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 142.15 }, "details": { "90% CI": "138.39 to 145.09", "Model release date": "2025-05-06", "Evaluations": "8 model-by-benchmark observations", "Versions": "gemini-2.5-pro-preview-05-06", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Aider polyglot; Fiction.LiveBench; GPQA diamond; GeoBench; Lech Mazur Writing; MATH level 5; The Agent Company; VPCT", "Source references": "Fiction.live leaderboard; GeoBench leaderboard; TheAgentCompany leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-4b1189a696e8e9cc", "model": "Gemma 4 26B A4B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 141.88 }, "details": { "90% CI": "138.86 to 143.43", "Model release date": "2026-04-02", "Evaluations": "5 model-by-benchmark observations", "Versions": "gemma-4-26b-a4b; gemma-4-26b-a4b-it_minimal", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-2352da7280f1decc", "model": "o1", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 141.72 }, "details": { "90% CI": "139.71 to 142.94", "Model release date": "2024-12-17", "Evaluations": "18 model-by-benchmark observations", "Versions": "o1-2024-12-17_high; o1-2024-12-17_low; o1-2024-12-17_medium", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 12; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; Aider polyglot; CadEval; Chess Puzzles; DTBench; Fiction.LiveBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; METR Time Horizons; OTIS Mock AIME 2024-2025; SimpleBench; SimpleQA Verified; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; CadEval Dashboard; Fiction.live leaderboard; GeoBench leaderboard; SimpleBench Leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/; lechmazur/writing Github repository" } }, { "id": "epoch-66aeb8c46b2fd22a", "model": "Claude Sonnet 4", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 141.56 }, "details": { "90% CI": "139.00 to 143.07", "Model release date": "2025-05-22", "Evaluations": "22 model-by-benchmark observations", "Versions": "claude-sonnet-4-20250514; claude-sonnet-4-20250514_12K; claude-sonnet-4-20250514_16K; claude-sonnet-4-20250514_2K; claude-sonnet-4-20250514_32K; claude-sonnet-4-20250514_59K; claude-sonnet-4-20250514_unknown", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 16; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Cybench; DTBench; DeepResearch Bench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; GSO-Bench; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; METR Time Horizons; OSWorld; OTIS Mock AIME 2024-2025; SimpleBench; The Agent Company; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; Cybench leaderboard; Fiction.live leaderboard; GSO Leaderboard; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; OS World Website; SimpleBench Leaderboard; TheAgentCompany leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://evals.futuresearch.ai/#drb; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-8cd358cb16e966de", "model": "Gemini 2.5 Flash (May 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 141.43 }, "details": { "90% CI": "139.18 to 142.82", "Model release date": "2025-05-20", "Evaluations": "7 model-by-benchmark observations", "Versions": "gemini-2.5-flash-preview-05-20; gemini-2.5-flash-preview-05-20_16K; gemini-2.5-flash-preview-05-20_23K", "Evaluator provenance": "Epoch AI: 1; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Fiction.LiveBench; GeoBench; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "Fiction.live leaderboard; GeoBench leaderboard; WeirdML Leaderboard; https://aider.chat/docs/leaderboards/; https://arcprize.org/leaderboard" } }, { "id": "epoch-ed5f601030c3af10", "model": "Mistral Medium 3.5", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 141.36 }, "details": { "90% CI": "138.50 to 144.33", "Model release date": "2026-04-28", "Evaluations": "6 model-by-benchmark observations", "Versions": "mistral-medium-2604", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "DTBench; FrontierCode; LMCA; ProofBench; Surface Evolver Bench; WeirdML", "Source references": "https://cognition.com/frontiercode; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.vals.ai/benchmarks/proof_bench; https://yhenon.github.io/surface-evolver-llm-eval/" } }, { "id": "epoch-5ae4de4ed6faf969", "model": "DeepSeek-R1 (May 2025)", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 141.13 }, "details": { "90% CI": "139.05 to 142.66", "Model release date": "2025-05-28", "Evaluations": "12 model-by-benchmark observations", "Versions": "DeepSeek-R1-0528; chutes/DeepSeek-R1-0528", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; DeepResearch Bench; Fiction.LiveBench; GPQA diamond; Lech Mazur Writing; MATH level 5; METR Time Horizons; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "DeepResearchBench Leaderboard; Fiction.live leaderboard; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://arcprize.org/leaderboard; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-76e3dba66433c22f", "model": "Claude 3.7 Sonnet", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 141.03 }, "details": { "90% CI": "138.98 to 142.68", "Model release date": "2025-02-24", "Evaluations": "20 model-by-benchmark observations", "Versions": "claude-3-7-sonnet-20250219; claude-3-7-sonnet-20250219_12K; claude-3-7-sonnet-20250219_15K; claude-3-7-sonnet-20250219_16K; claude-3-7-sonnet-20250219_2K; claude-3-7-sonnet-20250219_64K; claude-3-7-sonnet-20250219_8K", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 14; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; CadEval; Cybench; DeepResearch Bench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; GPQA diamond; GSO-Bench; GeoBench; Lech Mazur Writing; MATH level 5; METR Time Horizons; OSWorld; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; The Agent Company; VPCT", "Source references": "ARC Prize Leaderboard; CadEval Dashboard; Cybench leaderboard; DeepResearchBench Leaderboard; Fiction.live leaderboard; GSO Leaderboard; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; OS World Website; SimpleBench Leaderboard; TheAgentCompany leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-c33b388f7058f505", "model": "Gemini 2.5 Flash (Jun 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 140.59 }, "details": { "90% CI": "138.32 to 142.03", "Model release date": "2025-06-17", "Evaluations": "7 model-by-benchmark observations", "Versions": "gemini-2.5-flash", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Balrog; DTBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; LMCA; SimpleBench; Terminal Bench", "Source references": "SimpleBench Leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.tbench.ai/leaderboard/terminal-bench/2.0" } }, { "id": "epoch-f5f807e1d0c7e936", "model": "Grok-3 mini", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 140.2 }, "details": { "90% CI": "137.66 to 141.83", "Model release date": "2025-06-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "grok-3-mini-beta_high; grok-3-mini-beta_low; grok-3-mini-beta_medium; grok-3-mini_high; grok-3-mini_low", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Fiction.LiveBench; FrontierMath-2025-02-28-Private; GPQA diamond; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "ARC Prize Leaderboard; Fiction.live leaderboard; WeirdML Leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-dfc325305b0caf80", "model": "o3-mini", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 140.2 }, "details": { "90% CI": "137.03 to 141.80", "Model release date": "2025-01-31", "Evaluations": "20 model-by-benchmark observations", "Versions": "o3-mini-2025-01-31_high; o3-mini-2025-01-31_low; o3-mini-2025-01-31_medium", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 11; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; CadEval; Chess Puzzles; Cybench; DTBench; Fiction.LiveBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; GSO-Bench; LMCA; Lech Mazur Writing; MATH level 5; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleBench; SimpleQA Verified; WeirdML", "Source references": "ARC Prize Leaderboard; CadEval Dashboard; Cybench leaderboard; Fiction.live leaderboard; GSO Leaderboard; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-75ba797cc1823241", "model": "Kimi K2 (Jul 2025)", "org": "Moonshot", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.97 }, "details": { "90% CI": "136.28 to 141.90", "Model release date": "2025-07-12", "Evaluations": "7 model-by-benchmark observations", "Versions": "Kimi-K2-Instruct", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Aider polyglot; Fiction.LiveBench; GSO-Bench; Lech Mazur Writing; SimpleBench; Terminal Bench; WeirdML", "Source references": "Aider LLM Leaderboards; Fiction.live leaderboard; SimpleBench Leaderboard; Terminal-Bench v2 Leaderboard; WeirdML Leaderboard; https://gso-bench.github.io/index.html; lechmazur/writing Github repository" } }, { "id": "epoch-7d564e0e1938a86b", "model": "Gemini 2.5 Flash (Apr 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.84 }, "details": { "90% CI": "135.83 to 141.90", "Model release date": "2025-04-17", "Evaluations": "8 model-by-benchmark observations", "Versions": "gemini-2.5-flash-preview-04-17; gemini-2.5-flash-preview-04-17 (16K thinking); gemini-2.5-flash-preview-04-17 (24K thinking)", "Evaluator provenance": "Epoch AI: 1; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; Aider polyglot; Fiction.LiveBench; GeoBench; Lech Mazur Writing; OTIS Mock AIME 2024-2025; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; Fiction.live leaderboard; GeoBench leaderboard; VPCT leaderboard; WeirdML Leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-cd40178d5b8c265b", "model": "DeepSeek-V3.1", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.79 }, "details": { "90% CI": "135.65 to 143.26", "Model release date": "2025-08-21", "Evaluations": "5 model-by-benchmark observations", "Versions": "DeepSeek-V3.1; DeepSeek-V3.1_thinking", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; Fiction.LiveBench; LMCA; SimpleBench; WeirdML", "Source references": "Fiction.live leaderboard; SimpleBench Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-82386b4b368b0ab8", "model": "gpt-oss-120b", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.79 }, "details": { "90% CI": "135.57 to 142.12", "Model release date": "2025-08-05", "Evaluations": "14 model-by-benchmark observations", "Versions": "chutes/gpt-oss-120b; gpt-oss-120b; gpt-oss-120b_high; openai/gpt-oss-120b_high", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 11; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "APEX-Agents; Aider polyglot; DTBench; Fiction.LiveBench; GPQA diamond; LMCA; Lech Mazur Writing; METR Time Horizons; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleBench; Surface Evolver Bench; Terminal Bench; WeirdML", "Source references": "Aider LLM Leaderboards; Fiction.live leaderboard; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; Terminal-Bench v2 Leaderboard; WeirdML Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://yhenon.github.io/surface-evolver-llm-eval/; lechmazur/writing Github repository" } }, { "id": "epoch-c2b78fd7004645be", "model": "Qwen3-30B-A3B-Thinking (Jul 2025)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.55 }, "details": { "90% CI": "135.42 to 140.96", "Model release date": "2025-07-30", "Evaluations": "4 model-by-benchmark observations", "Versions": "qwen3-30b-a3b-thinking-2507", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-3e3c04992a0d23a2", "model": "Qwen3.5-9B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.32 }, "details": { "90% CI": "136.13 to 140.83", "Model release date": "2026-02-24", "Evaluations": "5 model-by-benchmark observations", "Versions": "qwen3.5-9B; qwen3.5-9B_none", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; Terminal Bench", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://www.tbench.ai/leaderboard/terminal-bench/2.0" } }, { "id": "epoch-bc471781684e9030", "model": "GPT-5 nano", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.21 }, "details": { "90% CI": "134.11 to 141.20", "Model release date": "2025-08-07", "Evaluations": "17 model-by-benchmark observations", "Versions": "gpt-5-nano-2025-08-07_high; gpt-5-nano-2025-08-07_low; gpt-5-nano-2025-08-07_medium; gpt-5-nano-2025-08-07_minimal; gpt-5-nano-2025-08-07_unknown", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Chess Puzzles; DTBench; Fiction.LiveBench; FrontierMath-Tier-4-v2-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; MATH level 5; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; ProofBench; SimpleQA Verified; Terminal Bench; VPCT; WeirdML", "Source references": "Fiction.live leaderboard; VPCT leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://www.tbench.ai/leaderboard/terminal-bench/2.0; https://www.vals.ai/benchmarks/proof_bench" } }, { "id": "epoch-9753a222f0ee7079", "model": "Qwen3-235B-A22B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 139.15 }, "details": { "90% CI": "134.62 to 140.53", "Model release date": "2025-04-28", "Evaluations": "9 model-by-benchmark observations", "Versions": "Qwen3-235B-A22B; chutes/Qwen3-235B-A22B; qwen3-235b-a22b", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Aider polyglot; DTBench; Fiction.LiveBench; GPQA diamond; LMCA; Lech Mazur Writing; MATH level 5; SimpleBench; WeirdML", "Source references": "Aider LLM Leaderboards; Fiction.live leaderboard; SimpleBench Leaderboard; WeirdML Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; lechmazur/writing Github repository" } }, { "id": "epoch-f7d79f342a93114c", "model": "DeepSeek-R1", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 138.82 }, "details": { "90% CI": "136.04 to 140.21", "Model release date": "2025-01-20", "Evaluations": "12 model-by-benchmark observations", "Versions": "DeepSeek-R1", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 9; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Balrog; Fiction.LiveBench; GPQA diamond; Lech Mazur Writing; MATH level 5; METR Time Horizons; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "ARC Prize Leaderboard; Fiction.live leaderboard; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://balrogai.com/; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-738cb185a03b5ed2", "model": "Qwen3-235B-A22B-Instruct (Jul 2025)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 138.78 }, "details": { "90% CI": "135.49 to 140.82", "Model release date": "2025-07-25", "Evaluations": "7 model-by-benchmark observations", "Versions": "Qwen3-235B-A22B-Instruct-2507; parasail-qwen3-235b-a22b-instruct-2507", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; DTBench; Fiction.LiveBench; LMCA; WeirdML", "Source references": "Fiction.live leaderboard; WeirdML Leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-27af7c76b6457b76", "model": "Qwen3-32B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 138.42 }, "details": { "90% CI": "134.68 to 140.22", "Model release date": "2025-04-29", "Evaluations": "6 model-by-benchmark observations", "Versions": "Qwen3-32B; Qwen3-32B_none; chutes/Qwen3-32B", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Aider polyglot; DTBench; Fiction.LiveBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "Fiction.live leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-bf105335be0e76a7", "model": "Grok 3", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 138.21 }, "details": { "90% CI": "135.44 to 139.51", "Model release date": "2025-04-09", "Evaluations": "13 model-by-benchmark observations", "Versions": "grok-3; grok-3-beta", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Balrog; Fiction.LiveBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "ARC Prize Leaderboard; Fiction.live leaderboard; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://balrogai.com/; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-3efd78bbd3d35b21", "model": "Qwen3-14B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 138.15 }, "details": { "90% CI": "133.24 to 140.03", "Model release date": "2025-04-29", "Evaluations": "5 model-by-benchmark observations", "Versions": "chutes/Qwen3-14B; qwen3-14b; qwen3-14b_none", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; Fiction.LiveBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "Fiction.live leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-ca1fe4e9c1499d82", "model": "gpt-oss-20b", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 137.7 }, "details": { "90% CI": "132.20 to 139.58", "Model release date": "2025-08-05", "Evaluations": "6 model-by-benchmark observations", "Versions": "gpt-oss-20b; gpt-oss-20b_high; gpt-oss-20b_medium", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025; Terminal Bench; WeirdML", "Source references": "Terminal-Bench v2 Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-4ff8980a69dd2fb9", "model": "QwQ-32B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 137.5 }, "details": { "90% CI": "132.62 to 141.76", "Model release date": "2025-03-05", "Evaluations": "5 model-by-benchmark observations", "Versions": "QwQ-32B; QwQ-32B (16K thinking); chutes/QwQ-32B", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Aider polyglot; Fiction.LiveBench; GPQA diamond; Lech Mazur Writing; OTIS Mock AIME 2024-2025", "Source references": "Fiction.live leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-4b9dfc020070b400", "model": "Qwen3-30B-A3B-Instruct (Jul 2025)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 137.27 }, "details": { "90% CI": "131.99 to 139.52", "Model release date": "2025-07-29", "Evaluations": "4 model-by-benchmark observations", "Versions": "qwen3-30b-a3b-instruct-2507", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-cab28bc8feb26807", "model": "GPT-4.1", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 136.65 }, "details": { "90% CI": "133.57 to 138.34", "Model release date": "2025-04-14", "Evaluations": "18 model-by-benchmark observations", "Versions": "gpt-4.1-2025-04-14", "Evaluator provenance": "Epoch AI: 8; Benchmark creator: 10; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; CadEval; Chess Puzzles; DTBench; Fiction.LiveBench; FrontierMath-Tier-4-2025-07-01-Private; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; GeoBench; LMCA; MATH level 5; OTIS Mock AIME 2024-2025; SWE-Bench verified; SimpleBench; SimpleQA Verified; WeirdML", "Source references": "ARC Prize Leaderboard; CadEval Dashboard; Fiction.live leaderboard; GeoBench leaderboard; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-e48e11ec92338950", "model": "GPT-4.5", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 136.65 }, "details": { "90% CI": "133.68 to 138.72", "Model release date": "2025-02-27", "Evaluations": "12 model-by-benchmark observations", "Versions": "gpt-4.5-preview-2025-02-27", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 8; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Cybench; Fiction.LiveBench; GPQA diamond; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025; SimpleBench; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; Cybench leaderboard; Fiction.live leaderboard; SimpleBench Leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-155aaf9a3362ee55", "model": "Qwen3-30B-A3B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 136.11 }, "details": { "90% CI": "130.91 to 138.67", "Model release date": "2025-04-29", "Evaluations": "7 model-by-benchmark observations", "Versions": "Qwen3-30B-A3B; Qwen3-30B-A3B_none; chutes/Qwen3-30B-A3B", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; Fiction.LiveBench; GPQA diamond; LMCA; Lech Mazur Writing; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "Fiction.live leaderboard; WeirdML Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; lechmazur/writing Github repository" } }, { "id": "epoch-39aa6615487815b2", "model": "Qwen3-8B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 136.06 }, "details": { "90% CI": "129.92 to 138.29", "Model release date": "2025-04-28", "Evaluations": "5 model-by-benchmark observations", "Versions": "chutes/Qwen3-8B; qwen3-8b; qwen3-8b_none", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; Fiction.LiveBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "Fiction.live leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-e32033b21b810217", "model": "DeepSeek-V3 (Mar 2025)", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 135.85 }, "details": { "90% CI": "132.24 to 137.75", "Model release date": "2025-03-24", "Evaluations": "11 model-by-benchmark observations", "Versions": "DeepSeek-V3-0324; chutes/DeepSeek-V3-0324", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Aider polyglot; DTBench; Fiction.LiveBench; GPQA diamond; LMCA; Lech Mazur Writing; MATH level 5; METR Time Horizons; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "Fiction.live leaderboard; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-973c6eb0e0af7b3d", "model": "o1-mini", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 135.7 }, "details": { "90% CI": "132.36 to 137.17", "Model release date": "2024-09-12", "Evaluations": "11 model-by-benchmark observations", "Versions": "o1-mini-2024-09-12_high; o1-mini-2024-09-12_medium; o1-mini-2024-09-12_unknown", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 6; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Cybench; FrontierMath-2025-02-28-Private; GPQA diamond; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "ARC Prize Leaderboard; Cybench leaderboard; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-07972d0694ef91cd", "model": "Gemini 2.0 Flash Thinking (Jan 2025)", "org": "Google DeepMind,Google", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 135.24 }, "details": { "90% CI": "130.55 to 137.87", "Model release date": "2025-01-21", "Evaluations": "6 model-by-benchmark observations", "Versions": "gemini-2.0-flash-thinking-exp-01-21", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Aider polyglot; Fiction.LiveBench; GPQA diamond; Lech Mazur Writing; OTIS Mock AIME 2024-2025; SimpleBench", "Source references": "Fiction.live leaderboard; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-5e586978f0322cd7", "model": "Gemini 2.0 Pro", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 134.95 }, "details": { "90% CI": "131.23 to 136.68", "Model release date": "2025-02-05", "Evaluations": "4 model-by-benchmark observations", "Versions": "gemini-2.0-pro-exp-02-05", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Hosted access (no API)", "Benchmarks": "Aider polyglot; Fiction.LiveBench; GPQA diamond; MATH level 5", "Source references": "Fiction.live leaderboard; https://aider.chat/docs/leaderboards/" } }, { "id": "epoch-758431785b770f34", "model": "GPT-4.1 mini", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 134.89 }, "details": { "90% CI": "130.97 to 136.49", "Model release date": "2025-04-14", "Evaluations": "15 model-by-benchmark observations", "Versions": "gpt-4.1-mini-2025-04-14", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 8; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; CadEval; Chess Puzzles; DTBench; Fiction.LiveBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; MATH level 5; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; SimpleQA Verified; WeirdML", "Source references": "ARC Prize Leaderboard; CadEval Dashboard; Fiction.live leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-6399058e427b9fdb", "model": "Gemini 2.0 Flash (Feb 2025)", "org": "Google DeepMind,Google", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 134.62 }, "details": { "90% CI": "130.95 to 136.33", "Model release date": "2025-02-05", "Evaluations": "11 model-by-benchmark observations", "Versions": "gemini-2.0-flash-001", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI-2; CadEval; DTBench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; GPQA diamond; GeoBench; MATH level 5; OTIS Mock AIME 2024-2025; The Agent Company; WeirdML", "Source references": "CadEval Dashboard; Fiction.live leaderboard; GeoBench leaderboard; TheAgentCompany experiment results github; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-331b0eded1193c4f", "model": "Gemini 2.0 Flash (Dec 2024)", "org": "Google DeepMind,Google", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 134.58 }, "details": { "90% CI": "124.18 to 136.47", "Model release date": "2024-12-11", "Evaluations": "4 model-by-benchmark observations", "Versions": "gemini-2.0-flash-exp", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Aider polyglot; Lech Mazur Writing; MMLU; SimpleBench", "Source references": "SimpleBench Leaderboard; Stanford CRFM Leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-d6bbfa6dd181a9fd", "model": "o1-preview", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 134.57 }, "details": { "90% CI": "130.76 to 137.92", "Model release date": "2024-09-12", "Evaluations": "8 model-by-benchmark observations", "Versions": "o1-preview-2024-09-12", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; Cybench; GPQA diamond; MATH level 5; METR Time Horizons; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "ARC Prize Leaderboard; Cybench leaderboard; SimpleBench Leaderboard; https://htihle.github.io/weirdml.html; https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/" } }, { "id": "epoch-a81641cc350f288c", "model": "Mistral Medium 3", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 134.02 }, "details": { "90% CI": "130.38 to 135.36", "Model release date": "2025-05-07", "Evaluations": "6 model-by-benchmark observations", "Versions": "mistral-medium-2505", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; FrontierMath-2025-02-28-Private; GPQA diamond; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; lechmazur/writing Github repository" } }, { "id": "epoch-34b0b5ba4f5f8428", "model": "Gemini 2.5 Flash-Lite (Jun 2025)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 133.82 }, "details": { "90% CI": "130.27 to 136.17", "Model release date": "2025-06-17", "Evaluations": "4 model-by-benchmark observations", "Versions": "gemini-2.5-flash-lite-preview-06-17-thinking; gemini-2.5-flash-lite-preview-06-17_16K", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; Fiction.LiveBench; LMCA; WeirdML", "Source references": "Fiction.live leaderboard; WeirdML Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-f7869767be09bb48", "model": "Claude 3.5 Sonnet (October 2024)", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 133.43 }, "details": { "90% CI": "129.16 to 137.84", "Model release date": "2024-10-22", "Evaluations": "18 model-by-benchmark observations", "Versions": "claude-3-5-sonnet-20241022", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 12; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Aider polyglot; Balrog; CadEval; DTBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; GSO-Bench; GeoBench; Lech Mazur Writing; MATH level 5; METR Time Horizons; MMLU; OTIS Mock AIME 2024-2025; SimpleBench; The Agent Company; VPCT; WeirdML", "Source references": "CadEval Dashboard; GSO Leaderboard; GeoBench leaderboard; SimpleBench Leaderboard; Stanford CRFM Leaderboard; TheAgentCompany experiment results github; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html; https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/; lechmazur/writing Github repository" } }, { "id": "epoch-ec57e2350b07a0f8", "model": "Magistral Small 1.0", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 133.07 }, "details": { "90% CI": "129.04 to 134.34", "Model release date": "2025-06-10", "Evaluations": "4 model-by-benchmark observations", "Versions": "magistral-small-2506", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC-AGI; ARC-AGI-2; GPQA diamond; OTIS Mock AIME 2024-2025", "Source references": "Not individually reported" } }, { "id": "epoch-77b7de2e585e927c", "model": "Qwen2.5-Max", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 132.48 }, "details": { "90% CI": "128.60 to 135.53", "Model release date": "2025-01-25", "Evaluations": "7 model-by-benchmark observations", "Versions": "qwen-max-2025-01-25; qwen2.5-max", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 3; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Aider polyglot; Fiction.LiveBench; FrontierMath-2025-02-28-Private; GPQA diamond; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025", "Source references": "Fiction.live leaderboard; https://aider.chat/docs/leaderboards/; lechmazur/writing Github repository" } }, { "id": "epoch-4973eaa8e3ba92b1", "model": "DeepSeek-V3", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 132.3 }, "details": { "90% CI": "128.06 to 135.23", "Model release date": "2024-12-26", "Evaluations": "14 model-by-benchmark observations", "Versions": "DeepSeek-V3; DeepSeek-V3-Base", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 3; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; Aider polyglot; BBH; FrontierMath-2025-02-28-Private; GPQA diamond; HellaSwag; MATH level 5; METR Time Horizons; MMLU; OTIS Mock AIME 2024-2025; PIQA; SimpleBench; TriviaQA; Winogrande", "Source references": "Aider LLM Leaderboards; DeepSeek-V3 Technical Report; METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; Stanford CRFM Leaderboard" } }, { "id": "epoch-3ec769983e05c017", "model": "Llama 4 Maverick", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 132.22 }, "details": { "90% CI": "127.72 to 134.29", "Model release date": "2025-04-06", "Evaluations": "14 model-by-benchmark observations", "Versions": "Llama-4-Maverick-17B-128E-Instruct; Llama-4-Maverick-17B-128E-Instruct-FP8; chutes/Llama-4-Maverick-17B-128E-Instruct", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 10; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; DTBench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; GPQA diamond; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "Fiction.live leaderboard; GeoBench leaderboard; SimpleBench Leaderboard; https://aider.chat/docs/leaderboards/; https://arcprize.org/leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-3e803ec45306fa77", "model": "Gemini 1.5 Pro (Sept 2024)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 131.72 }, "details": { "90% CI": "127.54 to 133.24", "Model release date": "2024-09-24", "Evaluations": "11 model-by-benchmark observations", "Versions": "gemini-1.5-pro-002", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 7; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI-2; Balrog; CadEval; DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; SimpleBench; The Agent Company; WeirdML", "Source references": "CadEval Dashboard; SimpleBench Leaderboard; Stanford CRFM Leaderboard; TheAgentCompany experiment results github; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-1cec437ecaa9e7f3", "model": "Phi-4", "org": "Microsoft Research", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 130.48 }, "details": { "90% CI": "124.66 to 132.01", "Model release date": "2024-12-12", "Evaluations": "6 model-by-benchmark observations", "Versions": "phi-4", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 2; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Balrog; GPQA diamond; Lech Mazur Writing; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Phi-4 Technical Report; https://balrogai.com/; lechmazur/writing Github repository" } }, { "id": "epoch-bf47fc8162cff777", "model": "Grok-2 (Dec 2024)", "org": "xAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 130.47 }, "details": { "90% CI": "125.72 to 132.22", "Model release date": "2024-12-12", "Evaluations": "8 model-by-benchmark observations", "Versions": "grok-2-1212", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 4; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; FrontierMath-2025-02-28-Private; GPQA diamond; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "SimpleBench Leaderboard; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-80b515d484dfc642", "model": "Claude 3.5 Sonnet", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 130.0 }, "details": { "90% CI": "Fixed calibration anchor; interval not published", "Model release date": "2024-06-20", "Evaluations": "12 model-by-benchmark observations", "Versions": "claude-3-5-sonnet-20240620", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 5; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Cybench; DTBench; FrontierMath-2025-02-28-Private; FrontierMath-Tier-4-2025-07-01-Private; GPQA diamond; MATH level 5; METR Time Horizons; MMLU; OTIS Mock AIME 2024-2025; SimpleBench; VPCT; WeirdML", "Source references": "Cybench leaderboard; SimpleBench Leaderboard; Stanford CRFM Leaderboard; VPCT leaderboard; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html; https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/" } }, { "id": "epoch-563fecda77be9306", "model": "Gemma 3 27B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 129.97 }, "details": { "90% CI": "124.48 to 132.04", "Model release date": "2025-03-12", "Evaluations": "9 model-by-benchmark observations", "Versions": "chutes/Gemma-3-27b-It; gemma-3-27b-it", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 6; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Aider polyglot; DTBench; Fiction.LiveBench; GPQA diamond; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; OTIS Mock AIME 2024-2025", "Source references": "Fiction.live leaderboard; GeoBench leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; lechmazur/writing Github repository" } }, { "id": "epoch-64bba10c932518b2", "model": "Llama 4 Scout", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 129.63 }, "details": { "90% CI": "125.13 to 131.19", "Model release date": "2025-04-05", "Evaluations": "9 model-by-benchmark observations", "Versions": "Llama-4-Scout-17B-16E-Instruct; chutes/Llama-4-Scout-17B-16E Instruct", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 5; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC-AGI; ARC-AGI-2; DTBench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; GPQA diamond; LMCA; MATH level 5; OTIS Mock AIME 2024-2025", "Source references": "ARC Prize Leaderboard; Fiction.live leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-9d68db3d0b40a21b", "model": "GPT-4.1 nano", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 129.54 }, "details": { "90% CI": "123.33 to 131.82", "Model release date": "2025-04-14", "Evaluations": "13 model-by-benchmark observations", "Versions": "gpt-4.1-nano-2025-04-14", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 7; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Chess Puzzles; DTBench; Fiction.LiveBench; FrontierMath-2025-02-28-Private; GPQA diamond; LMCA; MATH level 5; OTIS Mock AIME 2024-2025; SimpleQA Verified; WeirdML", "Source references": "ARC Prize Leaderboard; Fiction.live leaderboard; https://aider.chat/docs/leaderboards/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-575da255007bd21d", "model": "Gemini 1.5 Flash (Sep 2024)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 129.33 }, "details": { "90% CI": "123.75 to 131.20", "Model release date": "2024-09-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "gemini-1.5-flash-002", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 4; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Balrog; DTBench; FrontierMath-2025-02-28-Private; GPQA diamond; GeoBench; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; PIQA; WeirdML", "Source references": "GeoBench leaderboard; Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-90d68f050e989758", "model": "GPT-4o (May 2024)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 128.99 }, "details": { "90% CI": "124.62 to 131.23", "Model release date": "2024-05-13", "Evaluations": "8 model-by-benchmark observations", "Versions": "gpt-4o-2024-05-13", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Balrog; DTBench; GPQA diamond; LMCA; MATH level 5; METR Time Horizons; MMLU; OTIS Mock AIME 2024-2025", "Source references": "METR - Measuring AI Ability to Complete Long Tasks; Stanford CRFM Leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-ef968264b3653fc6", "model": "Qwen2.5-72B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 128.97 }, "details": { "90% CI": "123.48 to 130.73", "Model release date": "2024-09-19", "Evaluations": "18 model-by-benchmark observations", "Versions": "Qwen2.5-72B; Qwen2.5-VL-72B-Instruct; qwen2.5-72b-instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 8; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; BBH; Balrog; DTBench; GPQA diamond; GeoBench; HellaSwag; LMCA; MATH level 5; METR Time Horizons; MMLU; OSWorld; OTIS Mock AIME 2024-2025; PIQA; The Agent Company; TriviaQA; WeirdML; Winogrande", "Source references": "DeepSeek-V3 Technical Report; GeoBench leaderboard; METR - Measuring AI Ability to Complete Long Tasks; OS World Website; TheAgentCompany experiment results github; WeirdML Leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-4342c29a8d74018b", "model": "GPT-4o (Aug 2024)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 128.83 }, "details": { "90% CI": "124.26 to 130.96", "Model release date": "2024-08-06", "Evaluations": "10 model-by-benchmark observations", "Versions": "gpt-4o-2024-08-06", "Evaluator provenance": "Epoch AI: 6; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Aider polyglot; CadEval; Chess Puzzles; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; SimpleBench; SimpleQA Verified", "Source references": "CadEval Dashboard; SimpleBench Leaderboard; Stanford CRFM Leaderboard; https://aider.chat/docs/leaderboards/" } }, { "id": "epoch-d11d79df11a495c7", "model": "Llama 3.1-405B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 128.77 }, "details": { "90% CI": "124.44 to 130.61", "Model release date": "2024-07-23", "Evaluations": "15 model-by-benchmark observations", "Versions": "Llama-3.1-405B; Llama-3.1-405B-Instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; Cybench; DTBench; GPQA diamond; HellaSwag; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; PIQA; SimpleBench; The Agent Company; TriviaQA; WeirdML; Winogrande", "Source references": "Cybench leaderboard; DeepSeek-V3 Technical Report; SimpleBench Leaderboard; Stanford CRFM Leaderboard; The Llama 3 Herd of Models; TheAgentCompany experiment results github; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-af53a6c94c4fb6e1", "model": "GPT-4o (Nov 2024)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 128.69 }, "details": { "90% CI": "123.90 to 131.00", "Model release date": "2024-11-20", "Evaluations": "17 model-by-benchmark observations", "Versions": "gpt-4o-2024-11-20", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 10; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI; ARC-AGI-2; Aider polyglot; Cybench; FrontierMath-2025-02-28-Private; GDPval; GPQA diamond; GSO-Bench; GeoBench; Lech Mazur Writing; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; SWE-Bench verified; The Agent Company; VPCT; WeirdML", "Source references": "ARC Prize Leaderboard; Cybench leaderboard; GSO Leaderboard; GeoBench leaderboard; Phi-4 Technical Report; TheAgentCompany experiment results github; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-3111275532134560", "model": "Mistral Large 2 (Nov 2024)", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 128.57 }, "details": { "90% CI": "124.02 to 130.60", "Model release date": "2024-11-18", "Evaluations": "5 model-by-benchmark observations", "Versions": "mistral-large-2411", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 1; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "DTBench; FrontierMath-2025-02-28-Private; GPQA diamond; MATH level 5; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-ed537354e0d3186f", "model": "Mistral Large 2 (Jul 2024)", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 127.57 }, "details": { "90% CI": "122.04 to 129.73", "Model release date": "2024-07-24", "Evaluations": "7 model-by-benchmark observations", "Versions": "mistral-large-2407", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "DTBench; GPQA diamond; Lech Mazur Writing; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; SimpleBench", "Source references": "SimpleBench Leaderboard; Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; lechmazur/writing Github repository" } }, { "id": "epoch-247df6a9b3f5c768", "model": "Mistral Small 3.1", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 127.53 }, "details": { "90% CI": "122.32 to 129.35", "Model release date": "2025-03-17", "Evaluations": "4 model-by-benchmark observations", "Versions": "mistral-small-2503; mistral-small-3.1-2503", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; GPQA diamond; MATH level 5; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-baf2fc4d8a416cb5", "model": "Llama 3.3 70B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 127.33 }, "details": { "90% CI": "122.51 to 129.38", "Model release date": "2024-12-06", "Evaluations": "10 model-by-benchmark observations", "Versions": "Llama-3.3-70B-Instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 6; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Balrog; DTBench; Fiction.LiveBench; GPQA diamond; LMCA; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "Fiction.live leaderboard; SimpleBench Leaderboard; Stanford CRFM Leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-71939200cbec1432", "model": "GPT-4 Turbo (Apr 2024)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 127.27 }, "details": { "90% CI": "122.35 to 128.78", "Model release date": "2024-04-09", "Evaluations": "8 model-by-benchmark observations", "Versions": "gpt-4-turbo-2024-04-09", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; MATH level 5; METR Time Horizons; MMLU; OTIS Mock AIME 2024-2025; SimpleBench; WeirdML", "Source references": "METR - Measuring AI Ability to Complete Long Tasks; SimpleBench Leaderboard; Stanford CRFM Leaderboard; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-62b10cda26959338", "model": "Claude 3.5 Haiku", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 127.07 }, "details": { "90% CI": "120.92 to 129.77", "Model release date": "2024-10-22", "Evaluations": "12 model-by-benchmark observations", "Versions": "claude-3-5-haiku-20241022", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 7; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Aider polyglot; Balrog; CadEval; DTBench; FrontierMath-2025-02-28-Private; GPQA diamond; GeoBench; Lech Mazur Writing; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "CadEval Dashboard; GeoBench leaderboard; Stanford CRFM Leaderboard; https://aider.chat/docs/leaderboards/; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-743ff0bb4499a890", "model": "Gemini 1.5 Pro (May 2024)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 126.94 }, "details": { "90% CI": "121.96 to 129.05", "Model release date": "2024-05-14", "Evaluations": "6 model-by-benchmark observations", "Versions": "gemini-1.5-pro-001", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "BBH; DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Gemini 1.5 Report; Gemini 1.5 report; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-84f682a9ff8f0a66", "model": "Claude 3 Opus", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 126.9 }, "details": { "90% CI": "122.38 to 129.96", "Model release date": "2024-02-29", "Evaluations": "13 model-by-benchmark observations", "Versions": "claude-3-opus-20240229", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 5; Model developer: 3; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "Chess Puzzles; Cybench; DTBench; GPQA diamond; LMCA; MATH level 5; METR Time Horizons; MMLU; OTIS Mock AIME 2024-2025; SimpleBench; SimpleQA Verified; WeirdML; Winogrande", "Source references": "Cybench leaderboard; SimpleBench Leaderboard; Stanford CRFM Leaderboard; The Claude 3 Model Family: Opus, Sonnet, Haiku; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/" } }, { "id": "epoch-0ced3491fe96b781", "model": "GPT-4o mini", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 126.46 }, "details": { "90% CI": "120.11 to 128.49", "Model release date": "2024-07-18", "Evaluations": "20 model-by-benchmark observations", "Versions": "gpt-4o-mini-2024-07-18", "Evaluator provenance": "Epoch AI: 7; Benchmark creator: 10; Model developer: 3; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC-AGI-2; Aider polyglot; Balrog; Chess Puzzles; DTBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; GSM8K; GeoBench; LMCA; Lech Mazur Writing; MATH level 5; MMLU; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; PIQA; SimpleBench; SimpleQA Verified; VPCT; WeirdML", "Source references": "GeoBench leaderboard; Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Phi-4 Technical Report; SimpleBench Leaderboard; VPCT leaderboard; https://aider.chat/docs/leaderboards/; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html; lechmazur/writing Github repository" } }, { "id": "epoch-f318cf0a8b23175d", "model": "GPT-4 Turbo (Nov 2023)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 126.43 }, "details": { "90% CI": "121.25 to 128.46", "Model release date": "2024-01-25", "Evaluations": "6 model-by-benchmark observations", "Versions": "gpt-4-0125-preview; gpt-4-turbo", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; LMCA; MATH level 5; METR Time Horizons; MMLU", "Source references": "METR - Measuring AI Ability to Complete Long Tasks; Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-1c7c63fd7bc3b67b", "model": "Llama 3.1-70B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 125.93 }, "details": { "90% CI": "120.60 to 128.23", "Model release date": "2024-07-23", "Evaluations": "9 model-by-benchmark observations", "Versions": "Llama-3.1-70B-Instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 5; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Balrog; DTBench; GPQA diamond; LMCA; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; The Agent Company; WeirdML", "Source references": "Stanford CRFM Leaderboard; TheAgentCompany experiment results github; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-2037ab2d7119eee5", "model": "GPT-4 (Mar 2023)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 125.74 }, "details": { "90% CI": "120.54 to 131.03", "Model release date": "2023-03-14", "Evaluations": "7 model-by-benchmark observations", "Versions": "gpt-4-0314; gpt-4-32k-0314", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 1; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "GPQA diamond; GSM8K; HellaSwag; METR Time Horizons; MMLU; OTIS Mock AIME 2024-2025; Winogrande", "Source references": "GPT-4 Technical Report; GPT-4 technical report; METR - Measuring AI Ability to Complete Long Tasks; The Falcon Series of Open Language Models" } }, { "id": "epoch-640d4c041bf97bc3", "model": "Llama 3.2 90B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 125.52 }, "details": { "90% CI": "120.14 to 127.23", "Model release date": "2024-09-24", "Evaluations": "6 model-by-benchmark observations", "Versions": "Llama-3.2-90B-Vision-Instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 2; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Balrog; GPQA diamond; GeoBench; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "GeoBench leaderboard; Stanford CRFM Leaderboard; https://balrogai.com/" } }, { "id": "epoch-2b19a0b588ecbd87", "model": "Qwen2-72B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 125.28 }, "details": { "90% CI": "119.09 to 126.90", "Model release date": "2024-06-07", "Evaluations": "6 model-by-benchmark observations", "Versions": "qwen2-72b-instruct", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "GPQA diamond; MATH level 5; METR Time Horizons; MMLU; The Agent Company; WeirdML", "Source references": "METR - Measuring AI Ability to Complete Long Tasks; Stanford CRFM Leaderboard; TheAgentCompany experiment results github; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-6124dbe437746ae0", "model": "DeepSeek-V2 (MoE-236B, May 2024)", "org": "DeepSeek", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 124.8 }, "details": { "90% CI": "118.76 to 127.84", "Model release date": "2024-05-07", "Evaluations": "7 model-by-benchmark observations", "Versions": "DeepSeek-V2", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; HellaSwag; MMLU; PIQA; TriviaQA; Winogrande", "Source references": "DeepSeek-V3 Technical Report" } }, { "id": "epoch-0c9d81f80b1d8e4e", "model": "Gemma 3 12B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 123.54 }, "details": { "90% CI": "114.23 to 128.84", "Model release date": "2025-03-12", "Evaluations": "4 model-by-benchmark observations", "Versions": "gemma-3-12b-it", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-b5e936131bd11217", "model": "GPT-4 (Jun 2023)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 123.09 }, "details": { "90% CI": "116.85 to 126.92", "Model release date": "2023-06-13", "Evaluations": "11 model-by-benchmark observations", "Versions": "gpt-4-0613", "Evaluator provenance": "Epoch AI: 4; Benchmark creator: 3; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "BBH; DTBench; GPQA diamond; GSM8K; LMCA; MATH level 5; MMLU; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; TriviaQA; WeirdML", "Source references": "Baichuan 2: Open Large-scale Language Models; PapersWithCode; Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-8c00fe4a42bfde35", "model": "Llama 3-70B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 123.01 }, "details": { "90% CI": "117.25 to 125.75", "Model release date": "2024-04-18", "Evaluations": "7 model-by-benchmark observations", "Versions": "Meta-Llama-3-70B; Meta-Llama-3-70B-Instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 3; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Cybench; DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; Winogrande", "Source references": "Cybench leaderboard; Stanford CRFM Leaderboard; The Llama 3 Herd of Models; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-e51f15be5a01fe40", "model": "Gemini 1.5 Flash (May 2024)", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 122.68 }, "details": { "90% CI": "116.30 to 124.77", "Model release date": "2024-05-23", "Evaluations": "6 model-by-benchmark observations", "Versions": "gemini-1.5-flash-001", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; GSM8K; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-573b5fccf7d6f1e8", "model": "Gemma 2 27B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 122.12 }, "details": { "90% CI": "115.54 to 124.22", "Model release date": "2024-06-24", "Evaluations": "6 model-by-benchmark observations", "Versions": "gemma-2-27b; gemma-2-27b-it", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 2; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "DTBench; GPQA diamond; LMCA; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-d1b3bb26237a52d5", "model": "Mistral Large", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 122.12 }, "details": { "90% CI": "114.75 to 124.74", "Model release date": "2024-02-26", "Evaluations": "5 model-by-benchmark observations", "Versions": "mistral-large-2402", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-f79c7c98dbdc5a9a", "model": "Mixtral 8x22B", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 122.04 }, "details": { "90% CI": "115.27 to 124.38", "Model release date": "2024-04-17", "Evaluations": "6 model-by-benchmark observations", "Versions": "Mixtral-8x22B-Instruct-v0.1; Mixtral-8x22B-v0.1; open-mixtral-8x22b", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Cybench; DTBench; GPQA diamond; MATH level 5; MMLU; WeirdML", "Source references": "Cybench leaderboard; Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-97606524c9a0b918", "model": "phi-3-small 7.4B", "org": "Microsoft", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 121.82 }, "details": { "90% CI": "114.35 to 124.62", "Model release date": "2024-04-23", "Evaluations": "8 model-by-benchmark observations", "Versions": "Phi-3-small-8k-instruct", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ANLI; ARC AI2; BBH; HellaSwag; MMLU; OpenBookQA; TriviaQA; Winogrande", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Phi-3 Technical Report: A Highly Capable Language Model Locally on Your PhonePhi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone" } }, { "id": "epoch-144179838c9797f2", "model": "phi-3-medium 14B", "org": "Microsoft", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 121.19 }, "details": { "90% CI": "114.34 to 124.41", "Model release date": "2024-04-23", "Evaluations": "10 model-by-benchmark observations", "Versions": "Phi-3-medium-128k-instruct", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ANLI; ARC AI2; BBH; GPQA diamond; HellaSwag; MATH level 5; MMLU; OpenBookQA; TriviaQA; Winogrande", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard" } }, { "id": "epoch-caccf59751b5a2b0", "model": "Claude 3 Sonnet", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 120.84 }, "details": { "90% CI": "113.46 to 123.98", "Model release date": "2024-02-29", "Evaluations": "7 model-by-benchmark observations", "Versions": "claude-3-sonnet-20240229", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 2; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; WeirdML; Winogrande", "Source references": "Stanford CRFM Leaderboard; The Claude 3 Model Family: Opus, Sonnet, Haiku; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-63a6b3d8282b7310", "model": "Claude Instant", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 120.21 }, "details": { "90% CI": "112.98 to 123.40", "Model release date": "2023-08-09", "Evaluations": "5 model-by-benchmark observations", "Versions": "claude-instant-1.1; claude-instant-1.2", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 1; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ARC AI2; DTBench; GSM8K; MMLU; TriviaQA", "Source references": "Model Card and Evaluations for Claude Models; Releasing Claude Instant 1.2; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-9dc70fc15725f637", "model": "Claude 2", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 120.19 }, "details": { "90% CI": "113.70 to 124.02", "Model release date": "2023-07-11", "Evaluations": "6 model-by-benchmark observations", "Versions": "claude-2.0", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; TriviaQA", "Source references": "Model Card and Evaluations for Claude Models; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-0cb7d49d01ffac62", "model": "Gemma 2 9B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 119.81 }, "details": { "90% CI": "111.79 to 122.25", "Model release date": "2024-06-24", "Evaluations": "6 model-by-benchmark observations", "Versions": "gemma-2-9b; gemma-2-9b-it", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 0; Model developer: 3; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "GPQA diamond; GSM8K; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; PIQA", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard" } }, { "id": "epoch-c933eda95cb870a1", "model": "Qwen2.5-Coder-32B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 119.45 }, "details": { "90% CI": "110.76 to 124.79", "Model release date": "2024-09-18", "Evaluations": "5 model-by-benchmark observations", "Versions": "Qwen2.5-Coder-32B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-177bbe15eaf7dea8", "model": "Claude 2.1", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 119.32 }, "details": { "90% CI": "111.94 to 121.84", "Model release date": "2023-11-21", "Evaluations": "5 model-by-benchmark observations", "Versions": "claude-2.1", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; MMLU; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-505064d9efe3613d", "model": "Command R+", "org": "Cohere,Cohere Labs (formerly Cohere for AI)", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 119.2 }, "details": { "90% CI": "110.30 to 125.03", "Model release date": "2024-08-30", "Evaluations": "4 model-by-benchmark observations", "Versions": "c4ai-command-r-plus-08-2024", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "DTBench; LMCA; MMLU; SimpleBench", "Source references": "SimpleBench Leaderboard; Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-ee621a8b8aa7baa0", "model": "Claude 3 Haiku", "org": "Anthropic", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 118.85 }, "details": { "90% CI": "110.60 to 121.51", "Model release date": "2024-03-07", "Evaluations": "9 model-by-benchmark observations", "Versions": "claude-3-haiku-20240307", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "CadEval; DTBench; GPQA diamond; LMCA; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; WeirdML; Winogrande", "Source references": "CadEval Dashboard; Stanford CRFM Leaderboard; The Claude 3 Model Family: Opus, Sonnet, Haiku; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-bbc2e4a9da389edb", "model": "Mistral NeMo", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 118.65 }, "details": { "90% CI": "111.10 to 121.37", "Model release date": "2024-07-18", "Evaluations": "6 model-by-benchmark observations", "Versions": "Mistral-Nemo-Base-2407; Mistral-Nemo-Instruct-2407; open-mistral-nemo-2407", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 2; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Balrog; DTBench; GPQA diamond; GSM8K; MATH level 5; PIQA", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; https://balrogai.com/; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-9d7101d42cd98502", "model": "Qwen2.5-7B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 118.54 }, "details": { "90% CI": "109.60 to 121.73", "Model release date": "2024-09-19", "Evaluations": "6 model-by-benchmark observations", "Versions": "qwen2.5-7b-instruct", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "Balrog; DTBench; GPQA diamond; LMCA; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Stanford CRFM Leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-39efac89f56a745e", "model": "GPT-3.5 Turbo (Nov 2023)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 118.53 }, "details": { "90% CI": "110.58 to 121.43", "Model release date": "2023-11-06", "Evaluations": "8 model-by-benchmark observations", "Versions": "gpt-3.5-turbo-1106", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "ANLI; ARC AI2; GPQA diamond; MATH level 5; MMLU; OpenBookQA; TriviaQA; Winogrande", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone" } }, { "id": "epoch-1f26e2996c013d0f", "model": "Mixtral 8x7B", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 118.44 }, "details": { "90% CI": "111.29 to 120.90", "Model release date": "2023-12-11", "Evaluations": "12 model-by-benchmark observations", "Versions": "Mixtral-8x7B-Instruct-v0.1; Mixtral-8x7B-v0.1; open-mixtral-8x7b", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 1; Model developer: 9; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ANLI; ARC AI2; DTBench; GPQA diamond; GSM8K; HellaSwag; MATH level 5; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Mixtral of Experts; Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-13171917da828fe5", "model": "Ministral 3B", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 118.04 }, "details": { "90% CI": "106.31 to 121.89", "Model release date": "2024-10-16", "Evaluations": "4 model-by-benchmark observations", "Versions": "ministral-3b-2410", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; LMCA; MATH level 5", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-419b5a79160e8b0f", "model": "Yi-34B", "org": "01.AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 117.33 }, "details": { "90% CI": "108.94 to 120.60", "Model release date": "2023-11-02", "Evaluations": "5 model-by-benchmark observations", "Versions": "Yi-34B; Yi-34B-Chat", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 0; Model developer: 3; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "BBH; GPQA diamond; GSM8K; MATH level 5; MMLU", "Source references": "Stanford CRFM Leaderboard; Yi: Open Foundation Models by 01.AI" } }, { "id": "epoch-a7ee06797b994a25", "model": "phi-3-mini 3.8B", "org": "Microsoft", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 117.28 }, "details": { "90% CI": "109.58 to 120.85", "Model release date": "2024-04-23", "Evaluations": "8 model-by-benchmark observations", "Versions": "Phi-3-mini-4k-instruct", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ANLI; ARC AI2; BBH; HellaSwag; MMLU; OpenBookQA; TriviaQA; Winogrande", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone" } }, { "id": "epoch-9d0fe7153f940917", "model": "Gemini 1.0 Pro", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 117.14 }, "details": { "90% CI": "109.33 to 119.92", "Model release date": "2023-12-13", "Evaluations": "5 model-by-benchmark observations", "Versions": "gemini-1.0-pro-001", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-bc077715ff1b36c3", "model": "Stable Beluga 2", "org": "Stability AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 117.02 }, "details": { "90% CI": "109.56 to 120.11", "Model release date": "2023-07-20", "Evaluations": "7 model-by-benchmark observations", "Versions": "StableBeluga2", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; PIQA", "Source references": "Qwen Technical Report" } }, { "id": "epoch-9811036ef60ac2e8", "model": "Llama 3.1-8B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 116.47 }, "details": { "90% CI": "105.92 to 121.64", "Model release date": "2024-07-23", "Evaluations": "10 model-by-benchmark observations", "Versions": "Llama-3.1-8B-Instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 4; Model developer: 3; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "Balrog; DTBench; GPQA diamond; GSM8K; LMCA; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; PIQA; WeirdML", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard; https://balrogai.com/; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-93a4a97cb00244a7", "model": "Llama 3-8B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 116.39 }, "details": { "90% CI": "107.51 to 119.25", "Model release date": "2024-04-18", "Evaluations": "10 model-by-benchmark observations", "Versions": "Meta-Llama-3-8B; Meta-Llama-3-8B-Instruct", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ANLI; ARC AI2; DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; OpenBookQA; TriviaQA; Winogrande", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard; The Llama 3 Herd of Models; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-9d7bfeeb7aa5e284", "model": "Qwen2.5-Coder-14B", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 116.31 }, "details": { "90% CI": "106.42 to 122.20", "Model release date": "2024-09-18", "Evaluations": "5 model-by-benchmark observations", "Versions": "Qwen2.5-Coder-14B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-46f6b2a9b113f9a6", "model": "Gemma 3 4B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 115.88 }, "details": { "90% CI": "97.39 to 123.59", "Model release date": "2025-03-12", "Evaluations": "4 model-by-benchmark observations", "Versions": "gemma-3-4b-it", "Evaluator provenance": "Epoch AI: 2; Benchmark creator: 2; Model developer: 0; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "DTBench; GPQA diamond; LMCA; OTIS Mock AIME 2024-2025", "Source references": "https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca" } }, { "id": "epoch-3acb97256e1b025f", "model": "GPT-3.5 Turbo (Jan 2024)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 115.68 }, "details": { "90% CI": "107.76 to 119.09", "Model release date": "2024-01-25", "Evaluations": "9 model-by-benchmark observations", "Versions": "gpt-3.5-turbo-0125", "Evaluator provenance": "Epoch AI: 5; Benchmark creator: 3; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "DTBench; FrontierMath-Tiers-1-3-v2-Private; GPQA diamond; LMCA; MATH level 5; MMLU; Mystery Game Puzzles; OTIS Mock AIME 2024-2025; WeirdML", "Source references": "Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench; https://conceptualreasoning.ai/lmca; https://htihle.github.io/weirdml.html" } }, { "id": "epoch-bd9e5e4e26753c49", "model": "PaLM 2-L", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 114.9 }, "details": { "90% CI": "106.62 to 128.35", "Model release date": "2023-05-17", "Evaluations": "4 model-by-benchmark observations", "Versions": "PaLM 2-L", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; HellaSwag; TriviaQA; Winogrande", "Source references": "PaLM 2 Technical Report; The Falcon Series of Open Language Models" } }, { "id": "epoch-2fcb99af608f1fb2", "model": "Llama 2-70B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 113.67 }, "details": { "90% CI": "104.62 to 117.19", "Model release date": "2023-07-18", "Evaluations": "14 model-by-benchmark observations", "Versions": "Llama-2-70b-chat; Llama-2-70b-chat-hf; Llama-2-70b-hf ", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; DTBench; GPQA diamond; GSM8K; HellaSwag; LAMBADA; MATH level 5; MMLU; OTIS Mock AIME 2024-2025; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford CRFM Leaderboard; Stanford HELM; The Falcon Series of Open Language Models; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-1f62478a719e5aaf", "model": "GPT-3.5 Turbo (Jun 2023)", "org": "OpenAI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 113.22 }, "details": { "90% CI": "105.21 to 117.60", "Model release date": "2023-06-13", "Evaluations": "4 model-by-benchmark observations", "Versions": "gpt-3.5-turbo-0613", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "API access", "Benchmarks": "BBH; GSM8K; MMLU; Winogrande", "Source references": "Baichuan 2: Open Large-scale Language Models; Stanford CRFM Leaderboard; Stanford HELM; The Falcon Series of Open Language Models" } }, { "id": "epoch-4f3305160b282324", "model": "Qwen2.5-Coder (7B)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 113.01 }, "details": { "90% CI": "102.88 to 119.17", "Model release date": "2024-09-18", "Evaluations": "5 model-by-benchmark observations", "Versions": "Qwen2.5-Coder-7B; Qwen2.5-Coder-7B-Instruct", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-3d811093c1bcf63d", "model": "Qwen-14B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 112.88 }, "details": { "90% CI": "104.23 to 117.40", "Model release date": "2023-09-24", "Evaluations": "6 model-by-benchmark observations", "Versions": "Qwen-14B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; GSM8K; LAMBADA; MMLU; PIQA", "Source references": "Nemotron-4 15B Technical Report; Qwen Technical Report" } }, { "id": "epoch-44d382e718d4669d", "model": "Mistral 7B v0.1", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 112.05 }, "details": { "90% CI": "103.07 to 115.96", "Model release date": "2023-09-27", "Evaluations": "10 model-by-benchmark observations", "Versions": "Mistral-7B-v0.1", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ANLI; ARC AI2; BBH; GSM8K; HellaSwag; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Gemma: Open Models Based on Gemini Research and Technology; Mixtral of Experts; Nemotron-4 15B Technical Report; Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford HELM; Training Compute-Optimal Large Language Models" } }, { "id": "epoch-e9879f2b69e6b7a5", "model": "Falcon-180B", "org": "Technology Innovation Institute", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 111.96 }, "details": { "90% CI": "102.24 to 119.55", "Model release date": "2023-09-06", "Evaluations": "8 model-by-benchmark observations", "Versions": "falcon-180B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; Winogrande", "Source references": "The Falcon Series of Open Language Models; Yi: Open Foundation Models by 01.AI" } }, { "id": "epoch-6618094e80322597", "model": "internlm-20b", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 111.92 }, "details": { "90% CI": "102.59 to 116.56", "Model release date": "2023-09-18", "Evaluations": "6 model-by-benchmark observations", "Versions": "internlm-20b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; PIQA", "Source references": "Qwen Technical Report; Yi: Open Foundation Models by 01.AI" } }, { "id": "epoch-f86a51caf6d91698", "model": "Gemma 7B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 111.82 }, "details": { "90% CI": "103.14 to 116.69", "Model release date": "2024-02-21", "Evaluations": "10 model-by-benchmark observations", "Versions": "gemma-7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ANLI; ARC AI2; BBH; GSM8K; HellaSwag; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Gemma: Open Models Based on Gemini Research and Technology; Nemotron-4 15B Technical Report; Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard" } }, { "id": "epoch-ecb654236fd17378", "model": "LLaMA-65B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 109.97 }, "details": { "90% CI": "100.74 to 114.32", "Model release date": "2023-02-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "LLaMA-65B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "LLaMA: Open and Efficient Foundation Language Models; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford HELM" } }, { "id": "epoch-b65628f3ce4ac7b1", "model": "Falcon 2 11B", "org": "Technology Innovation Institute", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 109.36 }, "details": { "90% CI": "100.65 to 115.81", "Model release date": "2024-05-09", "Evaluations": "4 model-by-benchmark observations", "Versions": "falcon-11b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "Falcon2-11B Technical Report" } }, { "id": "epoch-7902e4a44b04effd", "model": "Mistral 7B v0.3", "org": "Mistral AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 108.88 }, "details": { "90% CI": "98.25 to 112.60", "Model release date": "2024-05-27", "Evaluations": "5 model-by-benchmark observations", "Versions": "Mistral-7B-Instruct-v0.3; open-mistral-7b", "Evaluator provenance": "Epoch AI: 3; Benchmark creator: 1; Model developer: 1; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "DTBench; GPQA diamond; MATH level 5; MMLU; OTIS Mock AIME 2024-2025", "Source references": "Stanford CRFM Leaderboard; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-7603f5344a4b9ae2", "model": "DeepSeek-Coder-V2-Lite-Base", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 108.68 }, "details": { "90% CI": "98.16 to 113.84", "Model release date": "2024-06-13", "Evaluations": "4 model-by-benchmark observations", "Versions": "DeepSeek-Coder-V2-Lite-Base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-10b20bd715f214bf", "model": "PaLM 2-M", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 108.06 }, "details": { "90% CI": "99.50 to 116.54", "Model release date": "2023-05-17", "Evaluations": "5 model-by-benchmark observations", "Versions": "PaLM 2-M", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; HellaSwag; OpenBookQA; TriviaQA; Winogrande", "Source references": "PaLM 2 Technical Report; The Falcon Series of Open Language Models" } }, { "id": "epoch-5b3f58064285f70c", "model": "Phi-2", "org": "Microsoft", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 107.73 }, "details": { "90% CI": "86.28 to 113.51", "Model release date": "2023-12-12", "Evaluations": "8 model-by-benchmark observations", "Versions": "phi-2", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ANLI; ARC AI2; BBH; HellaSwag; MMLU; OpenBookQA; TriviaQA; Winogrande", "Source references": "Phi-3 Technical Report: A Highly Capable Language Model Locally on Your Phone; Stanford CRFM Leaderboard" } }, { "id": "epoch-7a3a9ec3ed42fb89", "model": "Qwen2.5-Coder-3B", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 107.52 }, "details": { "90% CI": "93.28 to 114.93", "Model release date": "2024-09-18", "Evaluations": "4 model-by-benchmark observations", "Versions": "Qwen2.5-Coder-3B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; GSM8K; HellaSwag; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-f79f2cde4ff1930d", "model": "Nemotron-4 15B", "org": "Nvidia", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 107.49 }, "details": { "90% CI": "97.66 to 113.12", "Model release date": "2024-02-26", "Evaluations": "7 model-by-benchmark observations", "Versions": "Nemotron-4 15B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unreleased", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; MMLU; PIQA; Winogrande", "Source references": "Nemotron-4 15B Technical Report" } }, { "id": "epoch-760168daa189ab7b", "model": "Yi-9B", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 107.39 }, "details": { "90% CI": "97.63 to 114.91", "Model release date": "2024-03-01", "Evaluations": "4 model-by-benchmark observations", "Versions": "Yi-9B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; HellaSwag; MMLU; Winogrande", "Source references": "Yi: Open Foundation Models by 01.AI" } }, { "id": "epoch-38f9752e97108ecb", "model": "LLaMA-33B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 107.18 }, "details": { "90% CI": "97.05 to 111.91", "Model release date": "2023-02-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "LLaMA-33B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "LLaMA: Open and Efficient Foundation Language Models; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford HELM" } }, { "id": "epoch-fb1c30106909d04c", "model": "Qwen-7B", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 106.59 }, "details": { "90% CI": "95.28 to 111.81", "Model release date": "2023-09-28", "Evaluations": "6 model-by-benchmark observations", "Versions": "Qwen-7B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; GSM8K; LAMBADA; MMLU; PIQA", "Source references": "Qwen Technical Report; Yi: Open Foundation Models by 01.AI" } }, { "id": "epoch-b1ee736f5aa8a741", "model": "PaLM 2-S", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 105.91 }, "details": { "90% CI": "95.19 to 113.65", "Model release date": "2023-05-17", "Evaluations": "5 model-by-benchmark observations", "Versions": "PaLM 2-S", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; HellaSwag; OpenBookQA; TriviaQA; Winogrande", "Source references": "PaLM 2 Technical Report; The Falcon Series of Open Language Models" } }, { "id": "epoch-58b0326930f156bc", "model": "Llama 2-13B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 105.58 }, "details": { "90% CI": "94.81 to 110.87", "Model release date": "2023-07-18", "Evaluations": "11 model-by-benchmark observations", "Versions": "Llama-2-13b; Llama-2-13b-chat", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 1; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; DTBench; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Gemma: Open Models Based on Gemini Research and Technology; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford CRFM Leaderboard; Stanford HELM; The Falcon Series of Open Language Models; https://conceptualreasoning.ai/dtbench" } }, { "id": "epoch-90c5926bf154f019", "model": "Llama 2-34B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 104.93 }, "details": { "90% CI": "94.39 to 110.05", "Model release date": "2023-07-18", "Evaluations": "8 model-by-benchmark observations", "Versions": "Llama-2-34b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unreleased", "Benchmarks": "ARC AI2; BBH; GSM8K; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Llama 2: Open Foundation and Fine-Tuned Chat Models; Nemotron-4 15B Technical Report; The Falcon Series of Open Language Models" } }, { "id": "epoch-1656134874aa21ba", "model": "StarCoder 2 15B", "org": "Hugging Face,ServiceNow,Nvidia,BigCode", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 104.74 }, "details": { "90% CI": "91.31 to 111.67", "Model release date": "2024-02-20", "Evaluations": "4 model-by-benchmark observations", "Versions": "starcoder2-15b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-2b30b1fab5d2c26c", "model": "Yi 6B", "org": "01.AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 104.5 }, "details": { "90% CI": "93.95 to 110.26", "Model release date": "2023-11-22", "Evaluations": "6 model-by-benchmark observations", "Versions": "Yi-6B; Yi-6B-Chat", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "Yi: Open Foundation Models by 01.AI" } }, { "id": "epoch-ba014c89f47a81f2", "model": "Falcon-40B", "org": "Technology Innovation Institute", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 104.17 }, "details": { "90% CI": "93.56 to 109.76", "Model release date": "2023-05-25", "Evaluations": "10 model-by-benchmark observations", "Versions": "falcon-40b; falcon-40b-instruct", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Falcon2-11B Technical Report; Falcon2-11B Technical ReportFalcon2-11B Technical Report; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford HELM; The Falcon Series of Open Language Models" } }, { "id": "epoch-a7cd092f018246f7", "model": "Baichuan2-13B", "org": "Baichuan", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 102.9 }, "details": { "90% CI": "91.82 to 110.11", "Model release date": "2023-09-06", "Evaluations": "7 model-by-benchmark observations", "Versions": "Baichuan-2-13B-Base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; PIQA", "Source references": "Baichuan 2: Open Large-scale Language Models; Nemotron-4 15B Technical Report; Qwen Technical Report" } }, { "id": "epoch-9f0112cacadbf0ce", "model": "Qwen2.5-Coder (1.5B)", "org": "Alibaba", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 102.71 }, "details": { "90% CI": "84.79 to 110.28", "Model release date": "2024-09-18", "Evaluations": "5 model-by-benchmark observations", "Versions": "Qwen2.5-Coder-1.5B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-0274c1a973d37372", "model": "internlm-7b", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 102.58 }, "details": { "90% CI": "91.10 to 109.12", "Model release date": "2023-07-05", "Evaluations": "7 model-by-benchmark observations", "Versions": "internlm-7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; PIQA", "Source references": "Qwen Technical Report" } }, { "id": "epoch-827eea01dd2cf63f", "model": "INTELLECT-1", "org": "Prime Intellect,Hugging Face,Arcee AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 100.54 }, "details": { "90% CI": "88.72 to 105.92", "Model release date": "2024-11-29", "Evaluations": "6 model-by-benchmark observations", "Versions": "INTELLECT-1-Instruct", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "INTELLECT-1 Technical Report" } }, { "id": "epoch-861189aefa28c880", "model": "MPT-30B", "org": "MosaicML", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 100.33 }, "details": { "90% CI": "89.18 to 105.95", "Model release date": "2023-06-22", "Evaluations": "8 model-by-benchmark observations", "Versions": "mpt-30b; mpt-30b-instruct", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; BBH; GSM8K; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford HELM" } }, { "id": "epoch-ca4b3ebe6a8d46b6", "model": "LLaMA-13B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 100.32 }, "details": { "90% CI": "89.05 to 106.42", "Model release date": "2023-02-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "LLaMA-13B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Baichuan 2: Open Large-scale Language Models; LLaMA: Open and Efficient Foundation Language Models; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford HELM" } }, { "id": "epoch-7393ec90f90a576f", "model": "Llama 2-7B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 98.61 }, "details": { "90% CI": "87.31 to 105.58", "Model release date": "2023-07-18", "Evaluations": "10 model-by-benchmark observations", "Versions": "Llama-2-7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Gemma: Open Models Based on Gemini Research and Technology; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford CRFM Leaderboard; The Falcon Series of Open Language Models" } }, { "id": "epoch-d7498cc53a99b044", "model": "chatglm2-6b", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 98.59 }, "details": { "90% CI": "79.04 to 105.31", "Model release date": "2023-06-24", "Evaluations": "7 model-by-benchmark observations", "Versions": "chatglm2-6b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; PIQA", "Source references": "Baichuan 2: Open Large-scale Language Models; Qwen Technical Report" } }, { "id": "epoch-d35276ef6d3d52ee", "model": "LLaMA-7B", "org": "Meta AI", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 96.62 }, "details": { "90% CI": "83.72 to 103.51", "Model release date": "2023-02-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "LLaMA-7B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Baichuan 2: Open Large-scale Language Models; LLaMA: Open and Efficient Foundation Language Models; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford HELM" } }, { "id": "epoch-8e2aa17b027dafde", "model": "Baichuan 2-7B", "org": "Baichuan", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 95.94 }, "details": { "90% CI": "83.46 to 104.70", "Model release date": "2023-09-20", "Evaluations": "6 model-by-benchmark observations", "Versions": "Baichuan-2-7B-Base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU", "Source references": "Baichuan 2: Open Large-scale Language Models; Qwen Technical Report" } }, { "id": "epoch-b8f80535022f7435", "model": "DeepSeek Coder 33B", "org": "DeepSeek,Peking University", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 95.94 }, "details": { "90% CI": "82.55 to 103.09", "Model release date": "2023-11-02", "Evaluations": "4 model-by-benchmark observations", "Versions": "deepseek-coder-33b-base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-75ec34af30b3b527", "model": "Falcon-7B", "org": "Technology Innovation Institute", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 94.7 }, "details": { "90% CI": "82.31 to 102.04", "Model release date": "2023-04-24", "Evaluations": "10 model-by-benchmark observations", "Versions": "falcon-7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Baichuan 2: Open Large-scale Language Models; Falcon2-11B Technical Report; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report; Stanford HELM; The Falcon Series of Open Language Models" } }, { "id": "epoch-bb051df20f0935ee", "model": "CodeQwen1.5-7B", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 94.46 }, "details": { "90% CI": "79.30 to 103.08", "Model release date": "2024-04-15", "Evaluations": "4 model-by-benchmark observations", "Versions": "CodeQwen1.5-7B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-4fdefbc7eab890f5", "model": "vicuna-13b-v1.1", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 94.25 }, "details": { "90% CI": "81.54 to 102.09", "Model release date": "2023-04-12", "Evaluations": "7 model-by-benchmark observations", "Versions": "vicuna-13b-v1.1", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 7; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; OpenBookQA; PIQA; Winogrande", "Source references": "Baichuan 2: Open Large-scale Language Models; Textbooks Are All You Need II: phi-1.5 technical report" } }, { "id": "epoch-68c7ec9e25fd52d0", "model": "MPT-7B", "org": "MosaicML", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 94.19 }, "details": { "90% CI": "80.29 to 101.46", "Model release date": "2023-05-05", "Evaluations": "10 model-by-benchmark observations", "Versions": "mpt-7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 10; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; LAMBADA; MMLU; OpenBookQA; PIQA; TriviaQA; Winogrande", "Source references": "Baichuan 2: Open Large-scale Language Models; Llama 2: Open Foundation and Fine-Tuned Chat Models; Qwen Technical Report" } }, { "id": "epoch-0010a3a7460bc4dc", "model": "Gemma 2B", "org": "Google DeepMind", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 93.79 }, "details": { "90% CI": "80.82 to 100.43", "Model release date": "2024-02-21", "Evaluations": "8 model-by-benchmark observations", "Versions": "gemma-2b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 8; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; BBH; GSM8K; HellaSwag; MMLU; PIQA; TriviaQA; Winogrande", "Source references": "Gemma: Open Models Based on Gemini Research and Technology" } }, { "id": "epoch-595559ed5a63f61f", "model": "StarCoder 2 7B", "org": "Hugging Face,ServiceNow,Nvidia,BigCode", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 93.14 }, "details": { "90% CI": "75.68 to 101.70", "Model release date": "2024-02-20", "Evaluations": "4 model-by-benchmark observations", "Versions": "starcoder2-7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-e5ac7fab22af9746", "model": "XGen-7B", "org": "Salesforce", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 92.98 }, "details": { "90% CI": "79.45 to 99.66", "Model release date": "2023-06-27", "Evaluations": "6 model-by-benchmark observations", "Versions": "xgen-7b-8k-base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; HellaSwag; MMLU; OpenBookQA; PIQA; Winogrande", "Source references": "XGen-7B Technical Report" } }, { "id": "epoch-df316c8dc10c9ded", "model": "Qwen-1_8B", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 92.5 }, "details": { "90% CI": "70.76 to 101.32", "Model release date": "2023-11-30", "Evaluations": "6 model-by-benchmark observations", "Versions": "Qwen-1_8B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; BBH; GSM8K; LAMBADA; MMLU; PIQA", "Source references": "Qwen Technical Report" } }, { "id": "epoch-25e2b262703e066b", "model": "open_llama_7b", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 91.2 }, "details": { "90% CI": "77.68 to 98.45", "Model release date": "2023-06-07", "Evaluations": "6 model-by-benchmark observations", "Versions": "open_llama_7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; HellaSwag; MMLU; OpenBookQA; PIQA; Winogrande", "Source references": "XGen-7B Technical Report" } }, { "id": "epoch-d2badc884007f87e", "model": "Phi-1.5", "org": "Microsoft", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 91.06 }, "details": { "90% CI": "64.55 to 100.49", "Model release date": "2023-09-11", "Evaluations": "5 model-by-benchmark observations", "Versions": "phi-1_5", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; HellaSwag; MMLU; OpenBookQA; Winogrande", "Source references": "Textbooks Are All You Need II: phi-1.5 technical report" } }, { "id": "epoch-ebcc2fda5ab45d16", "model": "Baichuan1-7B", "org": "Baichuan", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 90.0 }, "details": { "90% CI": "75.57 to 98.45", "Model release date": "2023-06-01", "Evaluations": "4 model-by-benchmark observations", "Versions": "Baichuan-7B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (non-commercial)", "Benchmarks": "BBH; GSM8K; MMLU; PIQA", "Source references": "Baichuan 2: Open Large-scale Language Models; Qwen Technical Report" } }, { "id": "epoch-5065688298781eba", "model": "RedPajama-INCITE-7B-Base", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 89.96 }, "details": { "90% CI": "74.63 to 98.00", "Model release date": "2023-05-04", "Evaluations": "6 model-by-benchmark observations", "Versions": "RedPajama-INCITE-7B-Base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; HellaSwag; MMLU; OpenBookQA; PIQA; Winogrande", "Source references": "XGen-7B Technical Report" } }, { "id": "epoch-ab0e32e9f03caaa4", "model": "Dolly 2.0-12b", "org": "Databricks", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 89.2 }, "details": { "90% CI": "72.18 to 97.66", "Model release date": "2023-04-11", "Evaluations": "6 model-by-benchmark observations", "Versions": "dolly-v2-12b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; HellaSwag; MMLU; OpenBookQA; PIQA; Winogrande", "Source references": "XGen-7B Technical Report" } }, { "id": "epoch-64cb0e9fa7bc274f", "model": "DeepSeek Coder 6.7B", "org": "DeepSeek,Peking University", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 89.13 }, "details": { "90% CI": "72.38 to 98.12", "Model release date": "2023-11-02", "Evaluations": "4 model-by-benchmark observations", "Versions": "deepseek-coder-6.7b-base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-1a6659c7e5bef263", "model": "StarCoder 2 3B", "org": "Hugging Face,ServiceNow,Nvidia,BigCode", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 88.26 }, "details": { "90% CI": "72.12 to 97.11", "Model release date": "2024-02-22", "Evaluations": "4 model-by-benchmark observations", "Versions": "starcoder2-3b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-484e76a5e3bd5876", "model": "Qwen2.5-Coder-0.5B", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 87.63 }, "details": { "90% CI": "60.63 to 99.82", "Model release date": "2024-09-18", "Evaluations": "5 model-by-benchmark observations", "Versions": "Qwen2.5-Coder-0.5B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; GSM8K; HellaSwag; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-01fc9fa0f3066840", "model": "Cerebras-GPT-13B", "org": "Cerebras Systems", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 82.66 }, "details": { "90% CI": "66.35 to 91.36", "Model release date": "2023-03-20", "Evaluations": "6 model-by-benchmark observations", "Versions": "Cerebras-GPT-13B", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 6; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (unrestricted)", "Benchmarks": "ARC AI2; HellaSwag; MMLU; OpenBookQA; PIQA; Winogrande", "Source references": "XGen-7B Technical Report" } }, { "id": "epoch-18efb43a3c74e6b9", "model": "DeepSeek Coder 1.3B", "org": "DeepSeek,Peking University", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 62.59 }, "details": { "90% CI": "42.36 to 76.18", "Model release date": "2023-11-02", "Evaluations": "4 model-by-benchmark observations", "Versions": "deepseek-coder-1.3b-base", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 4; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Open weights (restricted use)", "Benchmarks": "ARC AI2; GSM8K; MMLU; Winogrande", "Source references": "Qwen2.5-Coder Technical Report" } }, { "id": "epoch-db9ae694e33701d4", "model": "stablelm-tuned-alpha-7b", "org": "Not reported", "settings": "Publisher composite across settings", "date": null, "estimated": false, "status": "Fitted composite · mixed provenance", "url": "https://epoch.ai/eci", "metrics": { "eci": 54.81 }, "details": { "90% CI": "28.52 to 71.34", "Model release date": "2023-04-19", "Evaluations": "5 model-by-benchmark observations", "Versions": "stablelm-tuned-alpha-7b", "Evaluator provenance": "Epoch AI: 0; Benchmark creator: 0; Model developer: 5; Unclassified: 0", "Settings policy": "Best result per benchmark across model settings/providers; not a single fixed-effort run", "Accessibility": "Unknown", "Benchmarks": "ARC AI2; HellaSwag; OpenBookQA; PIQA; Winogrande", "Source references": "XGen-7B Technical Report" } } ] } ] }