{
 "dates": "1–4 November 2026",
 "machine": "Mac Studio, M5 Ultra, 96 GB",
 "headToHead": [
  {
   "label": "Qwen3.8 Flash Next, 2.75 bpw",
   "runtime": "sushi",
   "decode": 119.1,
   "prefill": 2339,
   "tools": "8/8",
   "pick": true
  },
  {
   "label": "Ornith 1.5 122B, 4 bit",
   "runtime": "oMLX",
   "decode": 104.7,
   "prefill": 1786,
   "tools": "8/8"
  },
  {
   "label": "Qwen3.8 Flash Next, 4 bit",
   "runtime": "oMLX",
   "decode": 93.2,
   "prefill": 1356,
   "tools": "8/8"
  },
  {
   "label": "Qwen3.8 27B dense, 4 bit",
   "runtime": "oMLX",
   "decode": 74,
   "prefill": null,
   "tools": "8/8"
  }
 ],
 "contexts": [
  1000,
  16000,
  64000,
  128000
 ],
 "ladder": [
  {
   "label": "Flash Next 2.75 bpw, MTP",
   "runtime": "sushi",
   "pick": true,
   "decodeProse": [
    98.5,
    100.7,
    91.8,
    95.4
   ],
   "decodeCode": [
    142.4,
    141.1,
    132.4,
    133.7
   ],
   "prefill": [
    1652,
    2372,
    2297,
    2252
   ]
  },
  {
   "label": "Flash Next 2.75 bpw, DFlash",
   "runtime": "sushi",
   "pick": false,
   "decodeProse": [
    95.3,
    88.7,
    86.9,
    83.7
   ],
   "decodeCode": [
    144.9,
    145.3,
    137.2,
    134
   ],
   "prefill": [
    1649,
    2312,
    2264,
    2192
   ]
  },
  {
   "label": "Flash Next 4 bpw, MTP",
   "runtime": "sushi",
   "pick": false,
   "decodeProse": [
    103.1,
    103.2,
    99.6,
    90.9
   ],
   "decodeCode": [
    141.8,
    136.8,
    136.4,
    118.3
   ],
   "prefill": [
    1610,
    3401,
    3237,
    3150
   ]
  },
  {
   "label": "Flash Next 3.3 bpw, MTP",
   "runtime": "mlx-serve",
   "pick": false,
   "decodeProse": [
    99.2,
    93.3,
    82.5,
    78.8
   ],
   "decodeCode": [
    143.1,
    127.3,
    118.1,
    100
   ],
   "prefill": [
    563,
    1831,
    2128,
    2046
   ]
  },
  {
   "label": "DeepSeek V4.1 Flash 1.75 bpw",
   "runtime": "sushi",
   "pick": false,
   "decodeProse": [
    38.8,
    39.2,
    39.2,
    38.7
   ],
   "decodeCode": [
    36.7,
    40.7,
    37,
    38.5
   ],
   "prefill": [
    216,
    668,
    663,
    646
   ]
  }
 ],
 "quality": [
  {
   "label": "Flash Next 2.75 bpw",
   "runtime": "sushi",
   "tools": [
    29,
    32
   ],
   "checked": [
    16,
    16
   ],
   "code": [
    54,
    54
   ],
   "math": [
    48,
    48
   ],
   "recall": [
    12,
    12
   ],
   "read199k": 86.78,
   "grammar": [
    28,
    30
   ],
   "fixes": [
    8,
    8
   ]
  },
  {
   "label": "Flash Next 4 bpw",
   "runtime": "sushi",
   "tools": [
    30,
    32
   ],
   "checked": [
    16,
    16
   ],
   "code": [
    54,
    54
   ],
   "math": [
    48,
    48
   ],
   "recall": [
    12,
    12
   ],
   "read199k": 58.06,
   "grammar": [
    27,
    30
   ],
   "fixes": [
    8,
    8
   ]
  },
  {
   "label": "Flash Next 3.3 bpw",
   "runtime": "mlx-serve",
   "tools": [
    32,
    32
   ],
   "checked": [
    16,
    16
   ],
   "code": [
    54,
    54
   ],
   "math": [
    47,
    48
   ],
   "recall": [
    12,
    12
   ],
   "read199k": 89.07,
   "grammar": [
    26,
    30
   ],
   "fixes": [
    8,
    8
   ]
  },
  {
   "label": "DeepSeek V4.1 Flash 1.75 bpw",
   "runtime": "sushi",
   "tools": [
    27,
    32
   ],
   "checked": [
    14,
    16
   ]
  }
 ],
 "gguf": [
  {
   "label": "27B fine-tune \"Turbo 735\"",
   "runtime": "llama.cpp, 4 bit",
   "code": [
    36,
    36
   ],
   "math": [
    31,
    32
   ],
   "grammar": [
    28,
    30
   ],
   "fixes": [
    5,
    8
   ]
  },
  {
   "label": "27B fine-tune \"Jumbo 747\"",
   "runtime": "llama.cpp, 4 bit",
   "code": [
    35,
    36
   ],
   "math": [
    32,
    32
   ],
   "grammar": [
    27,
    30
   ],
   "fixes": [
    4,
    8
   ]
  }
 ],
 "blind": {
  "compared": {
   "final": "Qwen3.8-Flash-Next STQ 2.75 bpw on Sushi (server default temperature)",
   "iq33": "Qwen3.8-Flash-Next iQ 3.3 bpw on mlx-serve (T=0.6)"
  },
  "totalScore_outOf120": {
   "final": 98,
   "iq33": 97.5
  },
  "tasksWon": {
   "final": 7,
   "iq33": 3
  },
  "tasksTied": 2,
  "perTask_outOf10": [
   {
    "task": "telegram_backup",
    "final": 9,
    "iq33": 8.5
   },
   {
    "task": "mail_admin",
    "iq33": 7,
    "final": 7
   },
   {
    "task": "explain_beginner",
    "final": 7,
    "iq33": 9
   },
   {
    "task": "summary_logs",
    "iq33": 8,
    "final": 9
   },
   {
    "task": "formal_rewrite",
    "iq33": 8,
    "final": 9
   },
   {
    "task": "translate_tech",
    "iq33": 9,
    "final": 9
   },
   {
    "task": "client_apology",
    "final": 8,
    "iq33": 7.5
   },
   {
    "task": "daily_brief",
    "iq33": 8.5,
    "final": 7
   },
   {
    "task": "linkedin",
    "iq33": 9,
    "final": 8
   },
   {
    "task": "steps",
    "iq33": 7.5,
    "final": 8.5
   },
   {
    "task": "complaint_reply",
    "iq33": 7,
    "final": 7.5
   },
   {
    "task": "opinion",
    "final": 9,
    "iq33": 8.5
   }
  ]
 },
 "watchshop": [
  {
   "label": "STQ 2.75 bpw",
   "model": "Qwen3.8-Flash-Next",
   "runtime": "Sushi",
   "rubricTotal": 96,
   "features": 40,
   "mobile": 25,
   "accessibilityHygiene": 15,
   "design": 16,
   "buildTime_min": 11.6,
   "filesProduced": 24
  },
  {
   "label": "iQ 3.3 bpw",
   "model": "Qwen3.8-Flash-Next",
   "runtime": "mlx-serve 26.11.1",
   "rubricTotal": 97,
   "features": 40,
   "mobile": 24,
   "accessibilityHygiene": 15,
   "design": 18,
   "buildTime_min": 20.4,
   "filesProduced": 15
  }
 ],
 "pdf": [
  {
   "build": "STQ 2.75 bpw, T=0.6",
   "runtime": "Sushi",
   "seconds": 190,
   "passed": true
  },
  {
   "build": "STQ 2.75 bpw, T=0.6",
   "runtime": "Sushi",
   "seconds": 289,
   "passed": true
  },
  {
   "build": "STQ 2.75 bpw, T=1.0",
   "runtime": "Sushi",
   "seconds": 298,
   "passed": true
  },
  {
   "build": "STQ 2.75 bpw, T=1.0",
   "runtime": "Sushi",
   "seconds": 749,
   "passed": true
  },
  {
   "build": "4 bpw",
   "runtime": "Sushi",
   "seconds": 287,
   "passed": true
  },
  {
   "build": "4 bpw",
   "runtime": "Sushi",
   "seconds": 402,
   "passed": true
  },
  {
   "build": "iQ 3.3 bpw, T=0.6",
   "runtime": "mlx-serve 26.11.1",
   "seconds": 649,
   "passed": true
  },
  {
   "build": "iQ 3.3 bpw, T=0.6",
   "runtime": "mlx-serve 26.11.1",
   "seconds": 502,
   "passed": true
  }
 ],
 "temperature": [
  {
   "test": "Tool calls, 4 runs",
   "a": [
    29,
    32
   ],
   "b": [
    28,
    32
   ]
  },
  {
   "test": "Harder code tasks",
   "a": [
    18,
    18
   ],
   "b": [
    16,
    18
   ]
  },
  {
   "test": "Maths",
   "a": [
    48,
    48
   ],
   "b": [
    48,
    48
   ]
  }
 ],
 "memory": [
  {
   "label": "Flash Next 2.75 bpw (sushi)",
   "weights": 46.09
  },
  {
   "label": "Flash Next 3.3 bpw (mlx-serve)",
   "weights": 50.6
  }
 ],
 "harness": {
  "h2h-hermes-workload": {
   "script": "llmbench.py",
   "measures": "Streaming OpenAI-compatible chat requests; times are wall-clock from the client.",
   "sections": {
    "speed": "3 runs of a ~600-word French explanatory text (max 900 tokens, reasoning effort low).",
    "code": "3 runs of a Python module-writing prompt (max 900 tokens) — present only in later runs.",
    "long": "1 run: ~42k-token code corpus with one hidden fact (needle) in the middle; checks the fact is recalled.",
    "tools": "8 tool-calling cases (6 single-turn + 2 multi-turn follow-ups) with 4 tools (shell, web search, messaging, file read); auto-checked: correct tool, correct arguments, no tool when none needed, and refusing to restart a production service without confirmation.",
    "quality": "6–7 open questions (ops diagnosis, strict-format summary, scheduling logic, short French message, config bug, constraint scheduling, a planning question about a PDF signature task). Answers saved for manual grading; no grades are stored in the files."
   },
   "metrics": {
    "decode_tok_s": "(completion_tokens - 1) / (time of last streamed token - time of first streamed token)",
    "TTFT_s": "time to first streamed token (includes reasoning tokens)",
    "prefill_tok_s": "prompt_tokens / TTFT"
   }
  },
  "speed-ladder": {
   "script": "speedbench.py",
   "measures": "4 context rungs (1k, 16k, 64k, 128k tokens of English novel text, each prompt made unique to defeat prefix caching) × 2 tasks (French ~400-word summary = 'prose'; write a Python LRU-cache module = 'code') × 2 repetitions = 16 requests; max 1,200 output tokens, T=0.6, reasoning effort low.",
   "metrics": {
    "prefill_tok_s": "prompt_tokens / time to first token",
    "decode_tok_s": "(completion_tokens - 1) / streaming time"
   }
  },
  "tool-reliability-repeats": {
   "script": "tempbench.py",
   "measures": "4 runs × (8 tool-calling cases from llmbench + 4 quality questions with an automatic check: cron alert timing, constraint scheduling end time, the config bug fix, exactly-3-bullet summary).",
   "metrics": {
    "passed/total": "count of auto-checked cases passed"
   }
  },
  "hard-reasoning": {
   "script": "hardbench.py",
   "sections": {
    "math (tier 1)": "16 exact-answer maths/logic/date problems in French; final answer matched against a key.",
    "math2 (tier 2)": "16 harder counting/number-theory/logic problems (answers brute-forced in Python).",
    "code": "12 Python functions (e.g. roman numerals, interval merge, IBAN check, RPN, Dijkstra, LRU cache) run against hidden assert tests in a subprocess, 10 s timeout.",
    "code2": "6 harder functions (cron next-run, national-ID checksum, CSV line parser, topological sort, JSON path getter, sliding-window rate limiter) with hidden tests.",
    "long (tier 1)": "3 facts hidden in a ~85k-token code corpus + one sum combining two of them (4 checks).",
    "long2 (tier 2)": "5 notes hidden in a ~199k-token corpus, including a later correction that overrides an earlier value; 4 checks."
   },
   "samples": "3 samples per item for Sushi / mlx-serve runs, 2 for the 27B runs"
  },
  "french-quality": {
   "script": "frbench.py + frscore.py",
   "sections": {
    "grammarMCQ": "30 multiple-choice items (past-participle agreement, subjunctive, homophones, idioms, conjugation); auto-scored.",
    "sentenceFix": "8 faulty sentences to correct; passes if the expected correction appears.",
    "writingTasks": "12 realistic writing tasks (short chat message, formal letter to an administration, plain-language explanation, morning report, formal rewrite, technical translation, customer apology, daily briefing, social-media post, step-by-step instructions for an elderly person, complaint reply, short opinion piece).",
    "blindGrading": "frscore.py shuffles two models' writing answers under neutral labels X/Y; a grader scored each answer out of 10 without knowing which model wrote it."
   }
  },
  "watchshop-agent": {
   "script": "watchshop/run_watch.sh + prompt.tmpl + rubric.md",
   "measures": "One French prompt: a static vanilla HTML/CSS/JS shop for a fictional watch brand (home, catalogue of 8+ watches, filters/sort/search, product detail, cart with persistence, validated checkout, responsive at 375/768/1440 px, accessibility), served from a local folder. The agent has shell, file and code tools, up to 150 turns and a 1-hour budget, one run per model.",
   "rubric": "100 points: features 40, mobile 25, accessibility and hygiene 15 (Lighthouse mobile mean of accessibility/best practices/SEO scaled to 10, plus 5 for no console errors or 404s), design 20 (one grader, from screenshots). Functional checks scripted in a real Chrome."
  },
  "pdf-signature-agent": {
   "script": "pdf_sig_task/hermes_run/run_sig.sh + score.py + CRITERIA.md",
   "measures": "The Hermes agent gets a 6-page fillable insurance renewal form, a screenshot containing a handwritten signature, and the client details to enter. It must fill 6 values, cut out the signature ink with a real transparent background, and place it in 3 signature boxes. Up to 80 turns, 40-minute budget, 2 runs per configuration.",
   "autoScorer": "score.py checks the 6 values appear in the PDF text, counts images centred in the signature zones, and checks each has a transparency mask (SMask). PASS needs ≥5/6 fields, ≥3 signature images and ≥3 transparent ones.",
   "manualRubric": "CRITERIA.md defines a 100-point visual rubric (completeness of the signature, no white box, placement, sharpness); no manual scores are stored."
  }
 }
}