# MiMo Pro on Eight Sparks: Scheduling, Rates and Regression Scores Read a13/a14 cold extraction logs, the queued eighth-request fix, full model/base-engine pins, a GSM8K subset and older-build needle checks. - Type: Performance report - Evidence: source_reported; site reproduced: false - Model family: MiMo - Model: XiaomiMiMo/MiMo-V2.6-Pro-RL - Original author: rhys101 - Original project: https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8 - Pinned document: https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md - Source revision: 2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10 - Source commit time (not experiment date): 2026-09-24T18:54:01Z - Checked: 2026-10-18 - Experiment date: 2026-09-24 - Runtime topics: SGLang - Hardware topics: spark, nvidia - Reported setup: {"system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "kv_dtype": null, "context_limit_tokens": null, "experiment_date": "2026-09-24"} - Performance figures: attributed selections with per-point conditions; no cross-source ranking ## Conditions Unknown is null. Recipe options are not measured benchmark settings. ```json { "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "test_profiles": [ { "label": "coding C1", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L28", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "metric_denominator": "source per-stream decode" } }, { "label": "coding C8 per stream", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L28", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "metric_denominator": "source per-stream decode" } }, { "label": "eight categories C8 aggregate", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L35", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "metric_denominator": "all eight category output / total wall; not mean per-stream multiplied by 8" } }, { "label": "a13 cold CSV extraction C8", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/long-extraction-c8.txt#L4", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": "three waves, 8 requests per wave", "cache_state": "unique cache salt; prefix cache flushed before each wave", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "input_length_source_label": "four CSV fixtures, duplicated per wave; C8 input counts not retained", "output_tokens_cap": 4096, "actual_output_tokens_group": 43908, "group_wall_seconds": 187.18, "correct": 24, "total": 24, "seed": 20260922, "metric_denominator": "all completion usage tokens / total synchronized wave wall including prefill" } }, { "label": "a14 cold CSV extraction C8", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction-c8.txt#L4", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": "sha256:db156b3c950a0ec9c4a7d629f7574f65c1023150bcf2d0755c45d4f2c3cab661", "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a14", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": "three waves, 8 requests per wave", "cache_state": "unique cache salt; prefix cache flushed before each wave", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "input_length_source_label": "four CSV fixtures, duplicated per wave; C8 input counts not retained", "output_tokens_cap": 4096, "actual_output_tokens_group": 43900, "group_wall_seconds": 143.16, "correct": 24, "total": 24, "seed": 20260922, "metric_denominator": "all completion usage tokens / total synchronized wave wall including prefill" } }, { "label": "a14 cold CSV extraction C1", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction.txt#L13", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a14", "temperature": 0, "thinking": false, "input_tokens": 1498, "output_tokens": null, "repetitions": "four tasks x three repeats", "cache_state": "unique salt and cache flush before every request", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "output_tokens_cap": 4096, "actual_output_tokens_group": 21952, "group_wall_seconds": 209.86, "correct": 12, "total": 12, "seed": 20260922, "metric_denominator": "completion usage / summed client wall including prefill" } }, { "label": "a13 GSM8K first 200", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L83", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": null, "harness_default_concurrency": 8, "actual_harness_concurrency_reported": false, "benchmark": "first 200 GSM8K test questions in fixed order", "correct": 196, "total": 200, "shots": 0, "output_tokens_cap": 1024, "answer_extraction": "Answer: numeric, else last numeric; abs tolerance <1e-6", "wrong_ids": [ 93, 119, 162, 187 ], "a14_GSM8K_rerun": false, "leaderboard_equivalent": false } }, { "label": "a13 5K cold prefill", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L57", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": 3, "cache_state": "unique cold prefix", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "input_length_source_label": "5K", "statistic": "median", "metric_denominator": "TTFT based" } }, { "label": "a13 163K cold prefill", "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L58", "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": 3, "cache_state": "unique cold prefix", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "input_length_source_label": "163K", "statistic": "median", "metric_denominator": "TTFT based" } } ], "recipe_parameters": null, "deployment_options": [] } ``` ## Reading notes - Selected values are source excerpts, not independent experiment counts. ## Limits - No independent Run AI Here model test. - Read source date, exact execution identity, denominator, memory scope and retries per observation. - No unmatched quality ranking or hardware speed ranking. ## License declaration {"declared_spdx": null, "scope": "Attribution and finite original factual interpretation only; models and dependencies retain their own terms."} ## Editorial answer The a14 scheduler change raises cold eight-request extraction aggregate rate from 234.57 to 306.66 tokens/s while all 24 answers pass in each arm. This is long output, not a 128K input test: the single-request fixture log has 1498 input tokens. a13 quality scores and older needle runs keep their original build labels. By Run AI Here editorial ยท 2026-10-18 Local Spark Qwen draft; Codex source audit and rewrite. No site model benchmark. ## What is the eight-node recipe? The checkpoint is XiaomiMiMo/MiMo-V2.6-Pro-RL at full revision 73875d00b30a89ef8cc353a0b60b0e9f9561952d. Eight GB10 nodes use TP8/EP1, two switched200G RoCEv2 subnets at MTU9000, packed MXFP4 experts on Marlin W4A16, DFlash8 adaptive verify and FA4. The base SGLang SHA and a14 local image ID are pinned separately; a complete patched execution-tree revision and registry digest remain unknown. Sources: https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md, https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/versions.lock.json ## What changes in the paired extraction logs? The a13 scheduler counts a finishing prefill request against slots it already holds, leaving the eighth request queued. Across three cold eight-request waves, a13 has43908 completion tokens/187.18s and a14 has 43900/143.16s. Both pass24/24. Arithmetic mean per-stream rate falls 39.46 to 38.4 while complete-wave aggregate rises 234.57 to 306.66, showing why the denominators must stay separate. Sources: https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/long-extraction-c8.txt, https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction-c8.txt, https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c8.py ## How long is long extraction? The C1 source log records 1498 prompt tokens for every fixture, with roughly 1829 output tokens. C8 logs omit prompt counts. Unique salts and prefix-cache flushes make these cold extraction requests, but they are not128K-plus input workloads. Separate a13 cold prefill medians use nominal5K/20K/81K/163K labels. Sources: https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction.txt, https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c1.py, https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md ## Which quality and long-context checks apply to which build? GSM8K first200 fixed-order questions gives 196 correct on a13, greedy/thinking off with output cap1024 and a custom numeric extractor. No a14 GSM8K rerun is reported. Three-fact needle results at 127803 and 244168 tokens belong to a5/a6, not a13/a14. Finite smoke checks include vision; audio/video and peak runtime RAM are unmeasured. Sources: https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/gsm8k200.json, https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/eval/gsm8k_eval.py, https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md ## Selected source-reported figures Each point retains its own configuration, measurement unit and source. No site reproduction or cross-source ranking. ```json [ { "label": "coding C1", "metric": "decode_rate", "value": 91.1, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "metric_denominator": "source per-stream decode" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 28, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L28" }, { "label": "coding C8 per stream", "metric": "decode_rate", "value": 42.3, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "metric_denominator": "source per-stream decode" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 28, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L28" }, { "label": "eight categories C8 aggregate", "metric": "aggregate_output_rate", "value": 190.8, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "metric_denominator": "all eight category output / total wall; not mean per-stream multiplied by 8" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 35, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L35" }, { "label": "a13 cold CSV extraction C8", "metric": "aggregate_output_rate", "value": 234.57, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": "three waves, 8 requests per wave", "cache_state": "unique cache salt; prefix cache flushed before each wave", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "input_length_source_label": "four CSV fixtures, duplicated per wave; C8 input counts not retained", "output_tokens_cap": 4096, "actual_output_tokens_group": 43908, "group_wall_seconds": 187.18, "correct": 24, "total": 24, "seed": 20260922, "metric_denominator": "all completion usage tokens / total synchronized wave wall including prefill" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-a13-long8", "source_document_sha256": "443ce4520c8472347ac496024483ff46ab1b9a6a3851803d64bf382b66a26f37", "source_line": 4, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/long-extraction-c8.txt#L4" }, { "label": "a14 cold CSV extraction C8", "metric": "aggregate_output_rate", "value": 306.66, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": "sha256:db156b3c950a0ec9c4a7d629f7574f65c1023150bcf2d0755c45d4f2c3cab661", "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a14", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": "three waves, 8 requests per wave", "cache_state": "unique cache salt; prefix cache flushed before each wave", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "input_length_source_label": "four CSV fixtures, duplicated per wave; C8 input counts not retained", "output_tokens_cap": 4096, "actual_output_tokens_group": 43900, "group_wall_seconds": 143.16, "correct": 24, "total": 24, "seed": 20260922, "metric_denominator": "all completion usage tokens / total synchronized wave wall including prefill" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-a14-long8", "source_document_sha256": "36b64993a9c32e4b08be0ee600aa743e3fe4c62c917a6ae3ca05f9560c42233a", "source_line": 4, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction-c8.txt#L4" }, { "label": "a14 cold CSV extraction C1", "metric": "aggregate_output_rate", "value": 104.6, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a14", "temperature": 0, "thinking": false, "input_tokens": 1498, "output_tokens": null, "repetitions": "four tasks x three repeats", "cache_state": "unique salt and cache flush before every request", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "output_tokens_cap": 4096, "actual_output_tokens_group": 21952, "group_wall_seconds": 209.86, "correct": 12, "total": 12, "seed": 20260922, "metric_denominator": "completion usage / summed client wall including prefill" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-a14-long1", "source_document_sha256": "aa85e6233aaf60ab708d6580efada76287b2d560b4cc65f76f72f1cd3f5ab0d3", "source_line": 13, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction.txt#L13" }, { "label": "a13 GSM8K first 200", "metric": "accuracy", "value": 98.0, "unit": "percent", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": null, "harness_default_concurrency": 8, "actual_harness_concurrency_reported": false, "benchmark": "first 200 GSM8K test questions in fixed order", "correct": 196, "total": 200, "shots": 0, "output_tokens_cap": 1024, "answer_extraction": "Answer: numeric, else last numeric; abs tolerance <1e-6", "wrong_ids": [ 93, 119, 162, 187 ], "a14_GSM8K_rerun": false, "leaderboard_equivalent": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 83, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L83" }, { "label": "a13 5K cold prefill", "metric": "prefill_rate", "value": 1880, "unit": "tokens/s", "statistic": "median of three source runs; no SD", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": 3, "cache_state": "unique cold prefix", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "input_length_source_label": "5K", "statistic": "median", "metric_denominator": "TTFT based" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 57, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L57" }, { "label": "a13 163K cold prefill", "metric": "prefill_rate", "value": 1499, "unit": "tokens/s", "statistic": "median of three source runs; no SD", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": 3, "cache_state": "unique cold prefix", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "input_length_source_label": "163K", "statistic": "median", "metric_denominator": "TTFT based" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 58, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L58" } ] ``` ## Supplemental source identities ```json { "supplemental_sources": [ { "document_id": "mimopro-versions", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "versions.lock.json", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/versions.lock.json", "document_sha256": "e67a69bc019e7dc5e1a9c5184190901ca147b5f585ac4d1605ca23fde2b1b20c", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-gsm8k", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a13/gsm8k200.json", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/gsm8k200.json", "document_sha256": "4a281c460a7fa3a07b41cf3856faeba767c229a6d43b874ee7ff5d6729865cff", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-a13-long8", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a13/long-extraction-c8.txt", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/long-extraction-c8.txt", "document_sha256": "443ce4520c8472347ac496024483ff46ab1b9a6a3851803d64bf382b66a26f37", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-a14-long8", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a14/long-extraction-c8.txt", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction-c8.txt", "document_sha256": "36b64993a9c32e4b08be0ee600aa743e3fe4c62c917a6ae3ca05f9560c42233a", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-a14-long1", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a14/long-extraction.txt", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction.txt", "document_sha256": "aa85e6233aaf60ab708d6580efada76287b2d560b4cc65f76f72f1cd3f5ab0d3", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-score-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "eval/gsm8k_eval.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/eval/gsm8k_eval.py", "document_sha256": "207a2f4d38a1870a876bd62f8b9070baaf2a07faccbf25ac9ffe194c64b53353", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-speed-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "bench/mimobench.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/mimobench.py", "document_sha256": "d0425c39d1e5ac0f768ea8a451e39a3ad552e8808c2a5c269061faea3bb0b918", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-long1-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "bench/long_extraction_c1.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c1.py", "document_sha256": "3012886b036a75e049296c0d9a02aff20898003c806821fd39e1b8405eb6aabd", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-long8-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "bench/long_extraction_c8.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c8.py", "document_sha256": "3c27bb057d58d7afef9fd137bb7a67bb9341a05c85e0f29947e12362b37c685a", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" } ] } ``` ## Paired source quality comparisons ```json { "quality_comparisons": [ { "benchmark": "GSM8K first 200, build a13", "correct": 196, "total": 200, "wrong_ids": [ 93, 119, 162, 187 ], "shots": 0, "thinking": false, "a14_rerun": false, "leaderboard_score": false, "full_test": false } ] } ``` ## memory observations ```json { "memory_observations": [ { "kind": "weight storage / rank fit description, not peak RAM", "weight_files_GB": 535, "weights_per_rank_GB": [ 66, 69 ], "peak_runtime_RAM": null, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 118, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L118" } ] } ``` ## long context cases ```json { "long_context_cases": [ { "actual_input_tokens": 127803, "builds": [ "a5", "a6" ], "three_facts": true, "current_a13_a14_rerun": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 84, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L84" }, { "actual_input_tokens": 244168, "builds": [ "a5" ], "three_facts": true, "current_a13_a14_rerun": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 84, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L84" } ] } ``` ## case reports ```json { "case_reports": [ { "id": "a13-eighth-stream-queued", "classification": "source_reported_scheduler_slot_accounting_bug", "initial_build": "a13", "fixed_build": "a14", "initial_C8_aggregate": 234.57, "fixed_C8_aggregate": 306.66, "initial_tokens": 43908, "fixed_tokens": 43900, "initial_wall_seconds": 187.18, "fixed_wall_seconds": 143.16, "initial_correct": 24, "fixed_correct": 24, "total": 24, "initial_mean_per_stream": 39.46, "fixed_mean_per_stream": 38.4, "initial_max_TTFT_seconds": 46.39, "fixed_max_TTFT_seconds": 8.17, "C1_fixture_input_tokens": 1498, "C8_input_counts_retained": false, "universal_long_context_fix": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 49, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L49" } ] } ``` ## quality comparisons ```json { "quality_comparisons": [ { "benchmark": "GSM8K first 200, build a13", "correct": 196, "total": 200, "wrong_ids": [ 93, 119, 162, 187 ], "shots": 0, "thinking": false, "a14_rerun": false, "leaderboard_score": false, "full_test": false } ] } ```