{"schema":2,"as_of":"2026-11-09T08:00:09.237745+00:00","model":{"id":"deepseek-v4.1-flash","name":"DeepSeek V4.1 Flash"},"openrouter":{"run_at":"2026-11-09T08:00:09.237745+00:00","standings":[{"id":"gpqa_diamond","label":"GPQA Diamond","hint":"Graduate-level science questions","model":"deepseek-v4.1-flash","model_name":"DeepSeek V4.1 Flash","score":0.9084,"run_score":0.9091,"rolling_runs":12,"rolling_days":32,"samples":"198 questions × 3 epochs","line":0.8866,"margin_pp":2.179,"delta_pp":0.209,"rank":19,"of":52,"line_note":null,"parity":true,"peers":[{"provider":"Tenstorrent","score":0.9281},{"provider":"Near AI","score":0.9234},{"provider":"DeepInfra","score":0.9228},{"provider":"DigitalOcean","score":0.9209},{"provider":"SiliconFlow","score":0.9186},{"provider":"PrimaLabs","score":0.9176},{"provider":"DekaLLM","score":0.9158},{"provider":"CoreWeave","score":0.915},{"provider":"Baidu","score":0.9131},{"provider":"Alibaba","score":0.9125},{"provider":"InferenceNet · provider","score":0.9116},{"provider":"Venice","score":0.9116},{"provider":"GMICloud","score":0.9114},{"provider":"Fireworks · ac9eda21","score":0.9108},{"provider":"Sail Research · 8f31c510","score":0.9108},{"provider":"InferenceNet · a74ca7c9","score":0.9091},{"provider":"StepFun","score":0.9091},{"provider":"StreamLake","score":0.9091},{"provider":"Fireworks · c857cf2a","score":0.908},{"provider":"Cohere","score":0.9074},{"provider":"Nebius","score":0.9074},{"provider":"River","score":0.9074},{"provider":"Wafer · 66d4851d","score":0.9074},{"provider":"Wafer · provider","score":0.9074},{"provider":"Modal","score":0.907},{"provider":"Relace · 485d24f3","score":0.9063},{"provider":"Morph","score":0.9061},{"provider":"Decart","score":0.9059},{"provider":"BaseTen · 007b95bb","score":0.9058},{"provider":"Makora","score":0.9054},{"provider":"InferenceNet · 4231aea9","score":0.9052},{"provider":"AtlasCloud","score":0.9046},{"provider":"Parasail","score":0.9046},{"provider":"Mistral","score":0.9043},{"provider":"Relace · 301a2173","score":0.904},{"provider":"PrimeIntellect","score":0.9034},{"provider":"Particle AI","score":0.9032},{"provider":"Together","score":0.9029},{"provider":"Sail Research · a79e9269","score":0.9024},{"provider":"Sail Research · provider","score":0.9015},{"provider":"Fireworks · 9f90ca83","score":0.9007},{"provider":"Cloudflare","score":0.9006},{"provider":"BaseTen · b7371050","score":0.8986},{"provider":"Ionstream","score":0.898},{"provider":"Crusoe","score":0.8973},{"provider":"Wafer · 913e3d9a","score":0.8973},{"provider":"Phala","score":0.8939},{"provider":"OpenInference","score":0.8917},{"provider":"ModelRun","score":0.8822},{"provider":"Reka","score":0.8818},{"provider":"Novita","score":0.876}],"run_at":"2026-11-08T08:04:12.259735+00:00","runs":[{"run_at":"2026-10-31T17:05:54Z","score":0.9091,"tasks":198},{"run_at":"2026-11-01T22:17:51Z","score":0.9024,"tasks":198},{"run_at":"2026-11-02T02:00:14Z","score":0.9209,"tasks":198},{"run_at":"2026-11-02T06:56:19Z","score":0.904,"tasks":198},{"run_at":"2026-11-03T01:05:30Z","score":0.9192,"tasks":198},{"run_at":"2026-11-03T02:00:06Z","score":0.904,"tasks":198},{"run_at":"2026-11-04T02:03:59Z","score":0.9175,"tasks":198},{"run_at":"2026-11-05T02:02:46Z","score":0.9108,"tasks":198},{"run_at":"2026-11-05T08:04:05Z","score":0.8939,"tasks":198},{"run_at":"2026-11-06T08:02:08Z","score":0.8973,"tasks":198},{"run_at":"2026-11-07T08:00:24Z","score":0.9125,"tasks":198},{"run_at":"2026-11-08T08:04:15Z","score":0.9091,"tasks":198}],"peers_since":null,"full_window_median":null},{"id":"gpqa_diamond","label":"GPQA Diamond","hint":"Graduate-level science questions","model":"kimi-k3.1-pro","model_name":"Kimi K3.1 Pro","score":0.9327,"run_score":0.9377,"rolling_runs":6,"rolling_days":32,"samples":"198 questions × 3 epochs","line":0.9153,"margin_pp":1.736,"delta_pp":0.336,"rank":10,"of":26,"line_note":"provisional","parity":true,"peers":[{"provider":"InferenceNet · bed5b2b6","score":0.9411},{"provider":"Sail Research · 68e8eb89","score":0.9397},{"provider":"Nebius","score":0.9394},{"provider":"Sail Research · provider","score":0.9394},{"provider":"Sail Research · c0a8a518","score":0.9362},{"provider":"InferenceNet · 4889efe8","score":0.9356},{"provider":"Decart","score":0.9327},{"provider":"DigitalOcean","score":0.9327},{"provider":"Moonshot AI · da2df19f","score":0.9327},{"provider":"Modal","score":0.9318},{"provider":"Parasail","score":0.931},{"provider":"AkashML","score":0.9293},{"provider":"Morph","score":0.9293},{"provider":"Phala · d6759daf","score":0.9293},{"provider":"Fireworks","score":0.9276},{"provider":"InferenceNet · provider","score":0.9276},{"provider":"PrimaLabs","score":0.9273},{"provider":"Moonshot AI · a3fafaac","score":0.9259},{"provider":"Phala · 44be9331","score":0.9242},{"provider":"Together","score":0.9242},{"provider":"Wafer · ef01e5c5","score":0.9226},{"provider":"InferenceNet · ff212546","score":0.9211},{"provider":"Relace","score":0.9192},{"provider":"DeepInfra","score":0.9158},{"provider":"Wafer · 9dc2f681","score":0.9158}],"run_at":"2026-11-08T08:04:12.259735+00:00","runs":[{"run_at":"2026-11-05T00:15:49Z","score":0.9276,"tasks":198},{"run_at":"2026-11-05T02:02:45Z","score":0.931,"tasks":198},{"run_at":"2026-11-05T08:04:05Z","score":0.9276,"tasks":198},{"run_at":"2026-11-06T08:02:08Z","score":0.936,"tasks":198},{"run_at":"2026-11-07T08:00:24Z","score":0.936,"tasks":198},{"run_at":"2026-11-08T08:04:15Z","score":0.9377,"tasks":198}],"peers_since":null,"full_window_median":null},{"id":"tau_bench_verified_airline","label":"τ²-bench Verified airline","hint":"Airline customer-service agent tasks","model":"deepseek-v4.1-flash","model_name":"DeepSeek V4.1 Flash","score":0.7302,"run_score":0.72,"rolling_runs":15,"rolling_days":32,"samples":"50 tasks × 3 epochs","line":0.6967,"margin_pp":3.352,"delta_pp":0.522,"rank":21,"of":49,"line_note":null,"parity":true,"peers":[{"provider":"River","score":0.7766},{"provider":"AtlasCloud","score":0.7733},{"provider":"Wafer · provider","score":0.7711},{"provider":"ModelRun","score":0.7667},{"provider":"BaseTen","score":0.7633},{"provider":"CoreWeave","score":0.7633},{"provider":"Fireworks · 9f90ca83","score":0.7556},{"provider":"Fireworks · c857cf2a","score":0.755},{"provider":"GMICloud","score":0.755},{"provider":"Decart","score":0.7533},{"provider":"Alibaba","score":0.7511},{"provider":"Parasail","score":0.75},{"provider":"Fireworks · ac9eda21","score":0.7433},{"provider":"Mistral","score":0.7411},{"provider":"Baidu","score":0.741},{"provider":"Near AI","score":0.74},{"provider":"DekaLLM","score":0.7333},{"provider":"Sail Research · a79e9269","score":0.7333},{"provider":"Sail Research · provider","score":0.7333},{"provider":"OpenInference","score":0.7311},{"provider":"Together","score":0.73},{"provider":"Wafer · 66d4851d","score":0.73},{"provider":"Modal","score":0.7267},{"provider":"StepFun","score":0.7256},{"provider":"Phala","score":0.7244},{"provider":"Cloudflare","score":0.7233},{"provider":"Morph","score":0.7233},{"provider":"SiliconFlow","score":0.7233},{"provider":"Nebius","score":0.7217},{"provider":"Particle AI","score":0.7217},{"provider":"Ionstream","score":0.7213},{"provider":"Reka","score":0.7213},{"provider":"DeepInfra","score":0.72},{"provider":"Wafer · 913e3d9a","score":0.72},{"provider":"Novita","score":0.7167},{"provider":"PrimeIntellect","score":0.7167},{"provider":"Relace · 485d24f3","score":0.7167},{"provider":"StreamLake","score":0.7167},{"provider":"Venice","score":0.7167},{"provider":"Sail Research · 8f31c510","score":0.7156},{"provider":"Crusoe","score":0.7133},{"provider":"InferenceNet · a74ca7c9","score":0.7133},{"provider":"Cohere","score":0.71},{"provider":"InferenceNet · provider","score":0.71},{"provider":"DigitalOcean","score":0.7067},{"provider":"InferenceNet · 4231aea9","score":0.7067},{"provider":"Relace · 301a2173","score":0.7067},{"provider":"Makora","score":0.704}],"run_at":"2026-11-09T08:00:09.237745+00:00","runs":[{"run_at":"2026-11-03T02:31:17Z","score":0.7333,"tasks":50},{"run_at":"2026-11-04T02:35:54Z","score":0.7067,"tasks":50},{"run_at":"2026-11-05T02:34:58Z","score":0.7267,"tasks":50},{"run_at":"2026-11-05T08:53:07Z","score":0.7133,"tasks":50},{"run_at":"2026-11-06T09:10:30Z","score":0.72,"tasks":50},{"run_at":"2026-11-06T17:23:30Z","score":0.7333,"tasks":50},{"run_at":"2026-11-06T17:24:28Z","score":0.74,"tasks":50},{"run_at":"2026-11-06T17:35:38Z","score":0.7333,"tasks":50},{"run_at":"2026-11-06T17:44:37Z","score":0.7333,"tasks":50},{"run_at":"2026-11-06T17:45:26Z","score":0.7267,"tasks":50},{"run_at":"2026-11-06T17:55:00Z","score":0.7333,"tasks":50},{"run_at":"2026-11-06T17:56:53Z","score":0.7667,"tasks":50},{"run_at":"2026-11-07T08:48:41Z","score":0.7267,"tasks":50},{"run_at":"2026-11-08T08:33:39Z","score":0.74,"tasks":50},{"run_at":"2026-11-09T08:01:37Z","score":0.72,"tasks":50}],"peers_since":"2026-10-21","full_window_median":0.7333},{"id":"tau_bench_verified_airline","label":"τ²-bench Verified airline","hint":"Airline customer-service agent tasks","model":"kimi-k3.1-pro","model_name":"Kimi K3.1 Pro","score":0.7552,"run_score":0.7733,"rolling_runs":7,"rolling_days":32,"samples":"50 tasks × 3 epochs","line":0.7156,"margin_pp":3.964,"delta_pp":0.194,"rank":11,"of":25,"line_note":"provisional","parity":true,"peers":[{"provider":"InferenceNet · bed5b2b6","score":0.7891},{"provider":"DigitalOcean","score":0.7867},{"provider":"Fireworks","score":0.7867},{"provider":"Moonshot AI · da2df19f","score":0.7867},{"provider":"Phala · d6759daf","score":0.7867},{"provider":"Nebius","score":0.7733},{"provider":"Relace","score":0.7733},{"provider":"Wafer · 9dc2f681","score":0.7667},{"provider":"Decart","score":0.76},{"provider":"Morph","score":0.76},{"provider":"InferenceNet · 4889efe8","score":0.7533},{"provider":"InferenceNet · provider","score":0.7533},{"provider":"Moonshot AI · a3fafaac","score":0.7533},{"provider":"Phala · 44be9331","score":0.75},{"provider":"AkashML","score":0.7467},{"provider":"Alibaba","score":0.7467},{"provider":"DeepInfra","score":0.7467},{"provider":"InferenceNet · ff212546","score":0.7467},{"provider":"Wafer · ef01e5c5","score":0.7467},{"provider":"PrimaLabs","score":0.7448},{"provider":"Parasail","score":0.7333},{"provider":"Together","score":0.7333},{"provider":"Modal","score":0.73},{"provider":"Sail Research","score":0.73}],"run_at":"2026-11-09T08:00:09.237745+00:00","runs":[{"run_at":"2026-11-05T00:51:13Z","score":0.7667,"tasks":50},{"run_at":"2026-11-05T03:30:26Z","score":0.7533,"tasks":50},{"run_at":"2026-11-05T08:39:16Z","score":0.7733,"tasks":50},{"run_at":"2026-11-06T09:44:18Z","score":0.74,"tasks":50},{"run_at":"2026-11-07T09:47:25Z","score":0.7267,"tasks":50},{"run_at":"2026-11-08T09:03:28Z","score":0.7533,"tasks":50},{"run_at":"2026-11-09T08:01:36Z","score":0.7733,"tasks":50}],"peers_since":null,"full_window_median":null}],"rule":{"current_days":32,"baseline_days":21,"export_days":90,"url":"https://openrouter.ai/docs/guides/routing/auto-exacto"},"peers_source":{"snapshot_date":"2026-11-12","url":"https://openrouter.ai/api/v1/datasets/exports/benchmark-runs/2026-11-12.csv"},"differences":[{"label":"Every benchmark","items":["not through openrouter.ai: parity proxy (Chat Completions to api.logits.com, reasoning replayed as plain reasoning_content)","concurrency 32 (harness default 8; scheduling only)"]},{"label":"GPQA Diamond","items":["epochs 3 (harness default 10; OpenRouter's published runs use 3)"]},{"label":"τ²-bench Verified airline","items":["epochs 3 (harness default 1; OpenRouter's published runs use 3)"]}],"results":[]},"logits":{"run_at":null,"results":[],"tool_calls":[]},"compat":{"run_at":null,"tasksets":[],"harnesses":[]},"method":{"harness_repo":"https://github.com/OpenRouterTeam/benchmark-harness","harness_commit":"c6a05aa324948d4fd66b89c2c7e8b9a3065ee01a"}}