{ "schema_version": "1.0.0", "record_type": "executable_static_query_examples", "checked_on": "2026-10-18", "query_execution_is_model_benchmark": false, "server_side_query_API": false, "inputs": { "catalog": { "url": "/catalog.json", "sha256": "169d3662ce74cf538adf13e375682dd472a2659dcb1e6ca61428dcf0d7e85449" }, "articles": { "url": "/articles.json", "sha256": "c0a50d7242e4a653f13cc48e0576be22d41a7465845a97787c7fd80fdff2895d" } }, "examples": [ { "id": "exact-1017", "title": "Find the exact 1017 deployment artifact", "needs": [ "catalog" ], "purpose": "Discovery IDs only; read the complete record before evaluating compatibility.", "expression": "catalog.records.filter(r => r.kind === 'deployment' && r.model_artifact?.id === 'deepseek-ai/DeepSeek-V4.1-Flash-1017').map(r => r.id)", "expected_output": [ "deepseek-v41-flash-vllm" ] }, { "id": "spark-sglang", "title": "Find Spark reports tagged with SGLang", "needs": [ "catalog" ], "purpose": "Source discovery; the tags do not certify support for every listed model.", "expression": "catalog.records.filter(r => r.kind === 'benchmark' && r.hardware_topics.includes('spark') && r.runtime_topics.includes('SGLang')).map(r => r.id)", "expected_output": [ "dgxarley", "spark-pair-moe-served-rate", "qwen38-flash-next-sglang-concurrency", "mimopro-tp8-sglang-scheduler" ] }, { "id": "mac-ttft", "title": "Keep Mac TTFT points with their full conditions", "needs": [ "catalog" ], "purpose": "Preserve each original point, including uncertainty and unknown output length.", "expression": "catalog.records.filter(r => r.id === 'qwen35-m4pro-20260306').flatMap(r => r.editorial.reported_results.filter(p => p.metric === 'ttft').map(p => ({record_id:r.id, source:r.source, ...p})))", "expected_output": [ { "record_id": "qwen35-m4pro-20260306", "source": { "publisher": "weklund", "url": "https://github.com/weklund/mlx-coding-bench", "repository": "weklund/mlx-coding-bench", "platform": "github", "revision": "5ae9b491f7a0539d5abef949eded6bd57785a981", "document_path": "measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md", "pinned_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "report_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "last_commit_at": "2026-07-01T19:16:36Z", "checked_on": "2026-10-17", "metadata_status": "checked" }, "label": "Qwen3.5-4B int4 first token", "metric": "ttft", "value": 6.14, "unit": "seconds", "statistic": "mean of 20 measurements", "uncertainty": { "standard_deviation": 0.25 }, "conditions": { "system": "Apple M4 Pro 10P+4E / 20 GPU cores / 64 GB unified memory", "node_count": 1, "parallelism": null, "kv_dtype": null, "context_limit_tokens": null, "experiment_date": "2026-03-06", "runtime": "MLX 0.31.0 / mlx-lm 0.30.7", "repetitions": 20, "warmup_iterations": 1, "concurrency": null, "model_revision": null, "cache_state": null, "model": "Qwen3.5-4B int4", "quantization": "int4", "input_tokens": 4096, "output_tokens": null }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md#L40", "source_document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3" }, { "record_id": "qwen35-m4pro-20260306", "source": { "publisher": "weklund", "url": "https://github.com/weklund/mlx-coding-bench", "repository": "weklund/mlx-coding-bench", "platform": "github", "revision": "5ae9b491f7a0539d5abef949eded6bd57785a981", "document_path": "measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md", "pinned_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "report_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "last_commit_at": "2026-07-01T19:16:36Z", "checked_on": "2026-10-17", "metadata_status": "checked" }, "label": "Qwen3.5-9B int4 first token", "metric": "ttft", "value": 10.88, "unit": "seconds", "statistic": "mean of 20 measurements", "uncertainty": { "standard_deviation": 0.61 }, "conditions": { "system": "Apple M4 Pro 10P+4E / 20 GPU cores / 64 GB unified memory", "node_count": 1, "parallelism": null, "kv_dtype": null, "context_limit_tokens": null, "experiment_date": "2026-03-06", "runtime": "MLX 0.31.0 / mlx-lm 0.30.7", "repetitions": 20, "warmup_iterations": 1, "concurrency": null, "model_revision": null, "cache_state": null, "model": "Qwen3.5-9B int4", "quantization": "int4", "input_tokens": 4096, "output_tokens": null }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md#L41", "source_document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3" }, { "record_id": "qwen35-m4pro-20260306", "source": { "publisher": "weklund", "url": "https://github.com/weklund/mlx-coding-bench", "repository": "weklund/mlx-coding-bench", "platform": "github", "revision": "5ae9b491f7a0539d5abef949eded6bd57785a981", "document_path": "measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md", "pinned_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "report_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "last_commit_at": "2026-07-01T19:16:36Z", "checked_on": "2026-10-17", "metadata_status": "checked" }, "label": "Qwen3.5-27B int4 first token", "metric": "ttft", "value": 36.57, "unit": "seconds", "statistic": "mean of 20 measurements", "uncertainty": { "standard_deviation": 0.99 }, "conditions": { "system": "Apple M4 Pro 10P+4E / 20 GPU cores / 64 GB unified memory", "node_count": 1, "parallelism": null, "kv_dtype": null, "context_limit_tokens": null, "experiment_date": "2026-03-06", "runtime": "MLX 0.31.0 / mlx-lm 0.30.7", "repetitions": 20, "warmup_iterations": 1, "concurrency": null, "model_revision": null, "cache_state": null, "model": "Qwen3.5-27B int4", "quantization": "int4", "input_tokens": 4096, "output_tokens": null }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md#L42", "source_document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3" }, { "record_id": "qwen35-m4pro-20260306", "source": { "publisher": "weklund", "url": "https://github.com/weklund/mlx-coding-bench", "repository": "weklund/mlx-coding-bench", "platform": "github", "revision": "5ae9b491f7a0539d5abef949eded6bd57785a981", "document_path": "measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md", "pinned_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "report_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "last_commit_at": "2026-07-01T19:16:36Z", "checked_on": "2026-10-17", "metadata_status": "checked" }, "label": "Qwen3.5-35B-A3B int4 first token", "metric": "ttft", "value": 5.52, "unit": "seconds", "statistic": "mean of 20 measurements", "uncertainty": { "standard_deviation": 0.1 }, "conditions": { "system": "Apple M4 Pro 10P+4E / 20 GPU cores / 64 GB unified memory", "node_count": 1, "parallelism": null, "kv_dtype": null, "context_limit_tokens": null, "experiment_date": "2026-03-06", "runtime": "MLX 0.31.0 / mlx-lm 0.30.7", "repetitions": 20, "warmup_iterations": 1, "concurrency": null, "model_revision": null, "cache_state": null, "model": "Qwen3.5-35B-A3B int4", "quantization": "int4", "input_tokens": 4096, "output_tokens": null }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md#L43", "source_document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3" } ] }, { "id": "mac-bf16-memory", "title": "Read the two bf16 memory selections in A04", "needs": [ "articles" ], "purpose": "Source-reported selections, potentially overlapping catalog points; not additional experiments.", "expression": "articles.articles.filter(a => a.id === 'A04').flatMap(a => a.reported_results.filter(p => p.metric === 'peak_memory' && p.conditions.weight_precision === 'bf16').map(p => ({article_id:a.id, sources:a.sources, ...p})))", "expected_output": [ { "article_id": "A04", "sources": [ { "publisher": "weklund", "url": "https://github.com/weklund/mlx-coding-bench", "repository": "weklund/mlx-coding-bench", "platform": "github", "revision": "5ae9b491f7a0539d5abef949eded6bd57785a981", "document_path": "measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md", "pinned_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "report_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "last_commit_at": "2026-07-01T19:16:36Z", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "qwen35-m4pro-20260306" } ], "label": "Qwen3.5-4B bf16 peak_memory", "metric": "peak_memory", "value": 10.2, "unit": "GiB", "statistic": "reported peak; no per-cell dispersion supplied", "uncertainty": null, "conditions": { "system": "Apple M4 Pro 10P+4E / 20 GPU cores / 64 GB unified memory", "node_count": 1, "runtime": "MLX 0.31.0 / mlx-lm 0.30.7", "experiment_date": "2026-03-06", "repetitions": 20, "warmup_iterations": 1, "weight_revision": null, "conversion_revision": null, "kv_dtype": null, "concurrency": null, "cache_state": null, "sampling": null, "runtime_execution_revision": null, "model": "Qwen3.5-4B", "weight_precision": "bf16", "input_tokens": 4096, "output_tokens": null }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md#L53", "source_document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "source_record_id": "qwen35-m4pro-20260306", "source_line": 53 }, { "article_id": "A04", "sources": [ { "publisher": "weklund", "url": "https://github.com/weklund/mlx-coding-bench", "repository": "weklund/mlx-coding-bench", "platform": "github", "revision": "5ae9b491f7a0539d5abef949eded6bd57785a981", "document_path": "measurements/llm_benchmarks/Apple_M4_Pro_10P+4E+20GPU_64GB/2026-03-06__19:47:24/REPORT.md", "pinned_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "report_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md", "document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "last_commit_at": "2026-07-01T19:16:36Z", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "qwen35-m4pro-20260306" } ], "label": "Qwen3.5-9B bf16 peak_memory", "metric": "peak_memory", "value": 19.7, "unit": "GiB", "statistic": "reported peak; no per-cell dispersion supplied", "uncertainty": null, "conditions": { "system": "Apple M4 Pro 10P+4E / 20 GPU cores / 64 GB unified memory", "node_count": 1, "runtime": "MLX 0.31.0 / mlx-lm 0.30.7", "experiment_date": "2026-03-06", "repetitions": 20, "warmup_iterations": 1, "weight_revision": null, "conversion_revision": null, "kv_dtype": null, "concurrency": null, "cache_state": null, "sampling": null, "runtime_execution_revision": null, "model": "Qwen3.5-9B", "weight_precision": "bf16", "input_tokens": 4096, "output_tokens": null }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/weklund/mlx-coding-bench/blob/5ae9b491f7a0539d5abef949eded6bd57785a981/measurements/llm_benchmarks/Apple_M4_Pro_10P%2B4E%2B20GPU_64GB/2026-03-06__19%3A47%3A24/REPORT.md#L54", "source_document_sha256": "ff21a1f8a2597867e191a376cea28af293d47dfff82bb423193009b05c7ef5a3", "source_record_id": "qwen35-m4pro-20260306", "source_line": 54 } ] }, { "id": "qwen-spark-profile", "title": "Return the exact Qwen launch profile and source documents", "needs": [ "articles" ], "purpose": "Artifact and configuration lookup; missing executed versions and image digests remain null.", "expression": "articles.articles.filter(a => a.id === 'A05').map(a => ({article_id:a.id, conditions:a.conditions, sources:a.sources, supplemental_sources:a.supplemental_sources, deployment_profiles:a.deployment_profiles.filter(p => p.model_artifact === 'RadixArk/Qwen3.8-Flash-Next-NVFP4')}))", "expected_output": [ { "article_id": "A05", "conditions": { "hardware": "2 x DGX Spark GB10 sm_121", "node_count": 2, "memory_per_node_gib": 121, "physical_interconnect_gbps": 200, "campaign_date": "2026-09-05", "prefix_cache": false, "output_tokens_per_request": 512, "ignore_eos": true, "warmup": "every workload/concurrency shape, full generation; each node has its own JIT cache", "repetitions": 3, "summary_statistic": "median of three campaigns", "served_rate_denominator": "all output tokens / whole request wall clock, including TTFT", "runtime_execution_immutable_digest": null, "shared_memory_pool_claim": false, "cross_model_ranking_allowed": false, "current_1017_reproduction_claim": false }, "sources": [ { "publisher": "WeZZard", "url": "https://github.com/WeZZard/dgx-spark-bench", "repository": "WeZZard/dgx-spark-bench", "platform": "github", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "README.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/README.md", "report_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/README.md", "document_sha256": "b815d32807147f9638b1abbeadc02af257c55bd1f7651a123e5a8090bebe7469", "last_commit_at": "2026-09-06T00:35:29Z", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "spark-pair-moe-served-rate" } ], "supplemental_sources": [ { "document_id": "spark-REPORT", "parent_record_id": "spark-pair-moe-served-rate", "publisher": "WeZZard", "repository": "WeZZard/dgx-spark-bench", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "REPORT.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md", "document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "checked_on": "2026-10-18" }, { "document_id": "spark-docs-interconnect", "parent_record_id": "spark-pair-moe-served-rate", "publisher": "WeZZard", "repository": "WeZZard/dgx-spark-bench", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "docs/interconnect.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/docs/interconnect.md", "document_sha256": "9e7904db6d48fc44691c9f170c03f402c77480dcbf54f66bc107ef57ef0ea94f", "checked_on": "2026-10-18" }, { "document_id": "spark-docs-speculation", "parent_record_id": "spark-pair-moe-served-rate", "publisher": "WeZZard", "repository": "WeZZard/dgx-spark-bench", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "docs/speculation.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/docs/speculation.md", "document_sha256": "46c64755b468e6e996e84fd1d19727160a9130bb3565731e7b5f26185c1dfe7f", "checked_on": "2026-10-18" }, { "document_id": "spark-docs-engines", "parent_record_id": "spark-pair-moe-served-rate", "publisher": "WeZZard", "repository": "WeZZard/dgx-spark-bench", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "docs/engines.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/docs/engines.md", "document_sha256": "421632af313b5cfe7fca8b65ddabe0afb7b13e34229aca03e2af00c6a6f5452e", "checked_on": "2026-10-18" } ], "deployment_profiles": [ { "id": "qwen-v3", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "checkpoint_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "executed_engine_version": null, "image_digest": null, "parallelism": "TP2; two nodes", "context_limit_tokens": 120000, "configured_max_requests": null, "measured_concurrency": [ 1, 6 ], "kv_dtype": "bfloat16", "kv_pool_tokens": 161728, "speculation": "NEXTN; one step, top-k 1, draft tokens 2", "prefix_cache": false, "configuration_evidence": { "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L562", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_line": 562 }, "artifact_evidence": { "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L303", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_line": 303 }, "pool_evidence": { "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/README.md#L103", "source_document_sha256": "b815d32807147f9638b1abbeadc02af257c55bd1f7651a123e5a8090bebe7469", "source_document_id": "spark-pair-moe-served-rate", "source_line": 103 } } ] } ] }, { "id": "kv-pool-tradeoff", "title": "Read the A06 raw cells together with their derived summaries", "needs": [ "articles" ], "purpose": "Keep the model, conditions, raw point references and observed-range interpretation together.", "expression": "articles.articles.filter(a => a.id === 'A06').map(a => ({article_id:a.id, conditions:a.conditions, sources:a.sources, supplemental_sources:a.supplemental_sources, reported_results:a.reported_results, comparison_groups:a.comparison_groups}))", "expected_output": [ { "article_id": "A06", "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null }, "sources": [ { "publisher": "WeZZard", "url": "https://github.com/WeZZard/dgx-spark-bench", "repository": "WeZZard/dgx-spark-bench", "platform": "github", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "README.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/README.md", "report_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/README.md", "document_sha256": "b815d32807147f9638b1abbeadc02af257c55bd1f7651a123e5a8090bebe7469", "last_commit_at": "2026-09-06T00:35:29Z", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "spark-pair-moe-served-rate" } ], "supplemental_sources": [ { "document_id": "spark-REPORT", "parent_record_id": "spark-pair-moe-served-rate", "publisher": "WeZZard", "repository": "WeZZard/dgx-spark-bench", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "REPORT.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md", "document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "checked_on": "2026-10-18" }, { "document_id": "spark-docs-speculation", "parent_record_id": "spark-pair-moe-served-rate", "publisher": "WeZZard", "repository": "WeZZard/dgx-spark-bench", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "docs/speculation.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/docs/speculation.md", "document_sha256": "46c64755b468e6e996e84fd1d19727160a9130bb3565731e7b5f26185c1dfe7f", "checked_on": "2026-10-18" }, { "document_id": "spark-docs-engines", "parent_record_id": "spark-pair-moe-served-rate", "publisher": "WeZZard", "repository": "WeZZard/dgx-spark-bench", "revision": "25e8dbf1b6113dd0b3f12d27a670fb284944d260", "document_path": "docs/engines.md", "pinned_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/docs/engines.md", "document_sha256": "421632af313b5cfe7fca8b65ddabe0afb7b13e34229aca03e2af00c6a6f5452e", "checked_on": "2026-10-18" } ], "reported_results": [ { "id": "qwen-nospec-v3-short-c1-s1", "label": "qwen-nospec-v3-s1 short C1", "metric": "served_rate", "value": 26, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-nospec-v3-s1", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 600000, "speculative_decoding": "off", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L475", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 475 }, { "id": "qwen-nospec-v3-short-c1-s2", "label": "qwen-nospec-v3-s2 short C1", "metric": "served_rate", "value": 19.9, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-nospec-v3-s2", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 600000, "speculative_decoding": "off", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L476", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 476 }, { "id": "qwen-nospec-v3-short-c1-s3", "label": "qwen-nospec-v3-s3 short C1", "metric": "served_rate", "value": 22.6, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-nospec-v3-s3", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 600000, "speculative_decoding": "off", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L477", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 477 }, { "id": "qwen-v3-short-c1-s1", "label": "qwen-v3-s1 short C1", "metric": "served_rate", "value": 42.6, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-v3-s1", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 161728, "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L472", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 472 }, { "id": "qwen-v3-short-c1-s2", "label": "qwen-v3-s2 short C1", "metric": "served_rate", "value": 43, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-v3-s2", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 161728, "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L473", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 473 }, { "id": "qwen-v3-short-c1-s3", "label": "qwen-v3-s3 short C1", "metric": "served_rate", "value": 42.2, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-v3-s3", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 161728, "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L474", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 474 }, { "id": "qwen-nospec-v3-agentic-33k-c6-s1", "label": "qwen-nospec-v3-s1 agentic-33k C6", "metric": "served_rate", "value": 22.5, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-nospec-v3-s1", "campaign_repetitions": 3, "workload": "agentic-33k", "concurrency": 6, "reported_prompt_tokens": 30185, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 3072, "kv_pool_tokens": 600000, "speculative_decoding": "off", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L348", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 348 }, { "id": "qwen-nospec-v3-agentic-33k-c6-s2", "label": "qwen-nospec-v3-s2 agentic-33k C6", "metric": "served_rate", "value": 27.6, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-nospec-v3-s2", "campaign_repetitions": 3, "workload": "agentic-33k", "concurrency": 6, "reported_prompt_tokens": 30185, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 3072, "kv_pool_tokens": 600000, "speculative_decoding": "off", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L349", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 349 }, { "id": "qwen-nospec-v3-agentic-33k-c6-s3", "label": "qwen-nospec-v3-s3 agentic-33k C6", "metric": "served_rate", "value": 24.7, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-nospec-v3-s3", "campaign_repetitions": 3, "workload": "agentic-33k", "concurrency": 6, "reported_prompt_tokens": 30185, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 3072, "kv_pool_tokens": 600000, "speculative_decoding": "off", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L350", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 350 }, { "id": "qwen-v3-agentic-33k-c6-s1", "label": "qwen-v3-s1 agentic-33k C6", "metric": "served_rate", "value": 27.4, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-v3-s1", "campaign_repetitions": 3, "workload": "agentic-33k", "concurrency": 6, "reported_prompt_tokens": 30185, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 3072, "kv_pool_tokens": 161728, "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L345", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 345 }, { "id": "qwen-v3-agentic-33k-c6-s2", "label": "qwen-v3-s2 agentic-33k C6", "metric": "served_rate", "value": 28.6, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-v3-s2", "campaign_repetitions": 3, "workload": "agentic-33k", "concurrency": 6, "reported_prompt_tokens": 30185, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 3072, "kv_pool_tokens": 161728, "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L346", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 346 }, { "id": "qwen-v3-agentic-33k-c6-s3", "label": "qwen-v3-s3 agentic-33k C6", "metric": "served_rate", "value": 30.1, "unit": "tokens/s", "statistic": "one campaign cell; median is derived separately from three cells", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-v3-s3", "campaign_repetitions": 3, "workload": "agentic-33k", "concurrency": 6, "reported_prompt_tokens": 30185, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 3072, "kv_pool_tokens": 161728, "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L347", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 347 }, { "id": "qwen-nospec-v3-allocated-kv-pool", "label": "qwen-nospec-v3 allocated KV pool", "metric": "allocated_kv_pool", "value": 600000, "unit": "tokens", "statistic": "engine-allocated pool reported in the selected source cell; not maximum per-request context", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-nospec-v3-s1", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 600000, "speculative_decoding": "off", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L475", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 475 }, { "id": "qwen-v3-allocated-kv-pool", "label": "qwen-v3 allocated KV pool", "metric": "allocated_kv_pool", "value": 161728, "unit": "tokens", "statistic": "engine-allocated pool reported in the selected source cell; not maximum per-request context", "uncertainty": null, "conditions": { "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": "7b719225242aacd3dbd3f9407468c2ee9a9d2594", "weight_precision": "NVFP4", "system": "2 x DGX Spark GB10 sm_121; 121 GiB unified memory each", "node_count": 2, "parallelism": "TP2; 2 nodes", "physical_interconnect_gbps": 200, "reported_transport": "NCCL over RoCE", "experiment_date": "2026-09-05", "engine": "SGLang", "image_tag": "sglang-qwen38fn:sm121-ours", "image_digest": null, "runtime_execution_revision": null, "executed_engine_version": null, "kv_dtype": "bfloat16", "prefix_cache": false, "ignore_eos": true, "output_tokens_per_request": 512, "configured_max_total_tokens": 600000, "context_limit_tokens": 120000, "mem_fraction_static": 0.8, "warmup": "corrected campaign: every workload/user shape, full generation, per-node JIT cache", "clock_state": "author reports corrected underclock before this campaign; no independent traces here", "measurement_denominator": "all request output tokens / entire cell wall clock including TTFT", "temperature": null, "top_p": null, "configuration": "qwen-v3-s1", "campaign_repetitions": 3, "workload": "short", "concurrency": 1, "reported_prompt_tokens": 74, "prompt_token_count_scope": "REPORT prompt column; preserved as reported", "output_tokens_total": 512, "kv_pool_tokens": 161728, "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "failed_requests": 0 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L472", "source_document_sha256": "d08b9489656537f7b4a5d11cf4f07ed9f58c9b86ba6c7badd20ca52260a8224e", "source_document_id": "spark-REPORT", "source_record_id": "spark-pair-moe-served-rate", "source_line": 472 } ], "comparison_groups": [ { "id": "qwen-nospec-v3-short-c1", "workload": "short", "concurrency": 1, "arm": "qwen-nospec-v3", "speculative_decoding": "off", "kv_pool_tokens": 600000, "reported_prompt_tokens": 74, "output_tokens_total": 512, "output_tokens_per_request": 512, "samples": [ { "point_id": "qwen-nospec-v3-short-c1-s1", "value": 26, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L475" }, { "point_id": "qwen-nospec-v3-short-c1-s2", "value": 19.9, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L476" }, { "point_id": "qwen-nospec-v3-short-c1-s3", "value": 22.6, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L477" } ], "summary": { "median": 22.6, "minimum": 19.9, "maximum": 26, "unit": "tokens/s", "derivation": "computed from the three source cells", "interval_kind": "observed minimum and maximum, not SD or confidence interval" } }, { "id": "qwen-v3-short-c1", "workload": "short", "concurrency": 1, "arm": "qwen-v3", "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "kv_pool_tokens": 161728, "reported_prompt_tokens": 74, "output_tokens_total": 512, "output_tokens_per_request": 512, "samples": [ { "point_id": "qwen-v3-short-c1-s1", "value": 42.6, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L472" }, { "point_id": "qwen-v3-short-c1-s2", "value": 43, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L473" }, { "point_id": "qwen-v3-short-c1-s3", "value": 42.2, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L474" } ], "summary": { "median": 42.6, "minimum": 42.2, "maximum": 43, "unit": "tokens/s", "derivation": "computed from the three source cells", "interval_kind": "observed minimum and maximum, not SD or confidence interval" } }, { "id": "qwen-nospec-v3-agentic-33k-c6", "workload": "agentic-33k", "concurrency": 6, "arm": "qwen-nospec-v3", "speculative_decoding": "off", "kv_pool_tokens": 600000, "reported_prompt_tokens": 30185, "output_tokens_total": 3072, "output_tokens_per_request": 512, "samples": [ { "point_id": "qwen-nospec-v3-agentic-33k-c6-s1", "value": 22.5, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L348" }, { "point_id": "qwen-nospec-v3-agentic-33k-c6-s2", "value": 27.6, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L349" }, { "point_id": "qwen-nospec-v3-agentic-33k-c6-s3", "value": 24.7, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L350" } ], "summary": { "median": 24.7, "minimum": 22.5, "maximum": 27.6, "unit": "tokens/s", "derivation": "computed from the three source cells", "interval_kind": "observed minimum and maximum, not SD or confidence interval" } }, { "id": "qwen-v3-agentic-33k-c6", "workload": "agentic-33k", "concurrency": 6, "arm": "qwen-v3", "speculative_decoding": "NEXTN; one step; top-k 1; draft tokens 2", "kv_pool_tokens": 161728, "reported_prompt_tokens": 30185, "output_tokens_total": 3072, "output_tokens_per_request": 512, "samples": [ { "point_id": "qwen-v3-agentic-33k-c6-s1", "value": 27.4, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L345" }, { "point_id": "qwen-v3-agentic-33k-c6-s2", "value": 28.6, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L346" }, { "point_id": "qwen-v3-agentic-33k-c6-s3", "value": 30.1, "evidence_url": "https://github.com/WeZZard/dgx-spark-bench/blob/25e8dbf1b6113dd0b3f12d27a670fb284944d260/REPORT.md#L347" } ], "summary": { "median": 28.6, "minimum": 27.4, "maximum": 30.1, "unit": "tokens/s", "derivation": "computed from the three source cells", "interval_kind": "observed minimum and maximum, not SD or confidence interval" } } ] } ] }, { "id": "deepseek-1017-community", "title": "Read the exact 1017 TensorFold community report", "needs": [ "catalog" ], "purpose": "Keep the converted artifact, separate Engram tables, engine configuration and per-result cohort.", "expression": "catalog.records.filter(r => r.id === 'deepseek-1017-tensorfold-spark')", "expected_output": [ { "id": "deepseek-1017-tensorfold-spark", "title": "DeepSeek 1017: TensorFold on two DGX Sparks", "kind": "benchmark", "source_type": "community_operator_report", "summary": "A traceable 1017 EXL3 deployment and two named decode configurations, with a separate later prefill cohort and explicit correctness limits.", "runtime": "TensorFold / vLLM", "runtime_topics": [ "TensorFold", "vLLM" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "DeepSeek-V4.1-Flash-1017", "model_family": "DeepSeek", "model_id": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_release_date": null, "model_artifact": { "id": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1" }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "2 x DGX Spark GB10 TP2 over ConnectX-7", "node_count": 2, "parallelism": "TP2", "kv_dtype": null, "context_limit_tokens": null, "experiment_date": "2026-10-17 and 2026-10-18, separate cohorts" }, "test_profiles": [ { "label": "vllm-kit-run01 prose", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/01_vllm-miaai-kit_suite-results_32of32.json#L6", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "vllm-kit-run01", "engine": "vLLM + exllamav3 overlay", "engine_version": null, "runtime_execution_revision": null, "image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "kit_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "draft_tokens": 3, "configured_max_requests": 1, "context_limit_tokens": 262144, "prefill_chunk_tokens": 1536, "chat_template": "kit template; actual CHAT_TEMPLATE override not read by the source", "workload": "prose", "per_prompt_decode_rates": [ 24.7, 29.3 ], "per_prompt_TTFT_seconds": [ 0.32, 0.28 ], "prompt_count": 2 } }, { "label": "vllm-kit-run01 code", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/01_vllm-miaai-kit_suite-results_32of32.json#L20", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "vllm-kit-run01", "engine": "vLLM + exllamav3 overlay", "engine_version": null, "runtime_execution_revision": null, "image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "kit_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "draft_tokens": 3, "configured_max_requests": 1, "context_limit_tokens": 262144, "prefill_chunk_tokens": 1536, "chat_template": "kit template; actual CHAT_TEMPLATE override not read by the source", "workload": "code", "per_prompt_decode_rates": [ 40.4, 39.4 ], "per_prompt_TTFT_seconds": [ 0.47, 0.32 ], "prompt_count": 2 } }, { "label": "tensorfold-run11 prose", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/11_tensorfold-fd2365042f56_suite-results_32of32.json#L6", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "tensorfold-run11", "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": "fd2365042f56fcf726cf4071132c8b779ad2eae0", "image_tag": "tensorfold-dsv41:fd2365042f56", "image_digest": null, "draft_tokens": 5, "configured_max_requests": 4, "context_limit_tokens": 1048576, "prefill_chunk_tokens": 2048, "chat_template": "patched kit template; source infers per-run use from file times", "workload": "prose", "per_prompt_decode_rates": [ 53.4, 60.4 ], "per_prompt_TTFT_seconds": [ 0.12, 0.12 ], "prompt_count": 2 } }, { "label": "tensorfold-run11 code", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/11_tensorfold-fd2365042f56_suite-results_32of32.json#L20", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "tensorfold-run11", "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": "fd2365042f56fcf726cf4071132c8b779ad2eae0", "image_tag": "tensorfold-dsv41:fd2365042f56", "image_digest": null, "draft_tokens": 5, "configured_max_requests": 4, "context_limit_tokens": 1048576, "prefill_chunk_tokens": 2048, "chat_template": "patched kit template; source infers per-run use from file times", "workload": "code", "per_prompt_decode_rates": [ 91.1, 92.5 ], "per_prompt_TTFT_seconds": [ 0.13, 0.16 ], "prompt_count": 2 } }, { "label": "TensorFold final setup 32K prefill", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L16", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-18", "concurrency": 1, "thinking": false, "temperature": null, "top_p": null, "max_output_tokens": null, "output_tokens": null, "input_tokens": null, "prefix_cache": false, "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "prompt_tokens / TTFT; source final setup and host tuning", "repetitions": 5, "checkpoint_is_latest_revision_verified": false, "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": null, "runtime_revision_prefix": "92aa710b533d", "image_tag": "tensorfold-dsv41:92aa710b533d", "image_digest": null, "reported_prompt_bucket": "32K", "cached_tokens": 0, "per_run_values": [ 2201, 2180, 2065, 2165, 2100 ], "run_ids": [ "hw5", "hw6", "ht1", "ht2", "ht3" ], "draft_tokens": 5, "context_limit_tokens": 1048576, "warmup": "one 8K prompt; subsequent 8K requests before 32K/64K/128K; random-word prompts", "host_tuning_scope": "author applies no engine swap and memory-reclaim/compaction controls; not a universal host-tuning recommendation" } }, { "label": "TensorFold final setup 128K prefill", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L16", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-18", "concurrency": 1, "thinking": false, "temperature": null, "top_p": null, "max_output_tokens": null, "output_tokens": null, "input_tokens": null, "prefix_cache": false, "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "prompt_tokens / TTFT; source final setup and host tuning", "repetitions": 5, "checkpoint_is_latest_revision_verified": false, "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": null, "runtime_revision_prefix": "92aa710b533d", "image_tag": "tensorfold-dsv41:92aa710b533d", "image_digest": null, "reported_prompt_bucket": "128K", "cached_tokens": 0, "per_run_values": [ 1986, 1981, 1914, 1961, 1851 ], "run_ids": [ "hw5", "hw6", "ht1", "ht2", "ht3" ], "draft_tokens": 5, "context_limit_tokens": 1048576, "warmup": "one 8K prompt; subsequent 8K requests before 32K/64K/128K; random-word prompts", "host_tuning_scope": "author applies no engine swap and memory-reclaim/compaction controls; not a universal host-tuning recommendation" } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "inference_speed_and_task_checks", "notes": [ "The selected 1017 results describe two complete serving configurations, not an isolated engine-only test. Keep the converted weights and separate Engram revision, distinguish October 17 decode from October 18 prefill, and retain missing image, cache and quality evidence." ], "limitations": [ "Source-reported only; no independent site model runs.", "Document revision identifies the report, not the executed runtime or weight revision.", "Complete measured configurations and missing fields must accompany extracted values." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "Elevatormusic", "url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "platform": "github", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "README.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/README.md", "report_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/README.md", "document_sha256": "546a0c3bfbbb6e84226ec812cf38927f6d993e53b73633a540c426eea5f13d29", "last_commit_at": "2026-10-18T05:40:29Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "ds1017-docs-BENCHMARKS.md", "parent_record_id": "deepseek-1017-tensorfold-spark", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "docs/BENCHMARKS.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/BENCHMARKS.md", "document_sha256": "b2d05a077ae8145a2cf2a98ac2ae2eabef87b992230c32398a60e0a2e9174861", "checked_on": "2026-10-18" }, { "document_id": "ds1017-docs-KNOWN-LIMITS.md", "parent_record_id": "deepseek-1017-tensorfold-spark", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "docs/KNOWN-LIMITS.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/KNOWN-LIMITS.md", "document_sha256": "eeb6bc6f7a3fce20654f977581e9718b124e70fc01ab358efebefe9afd9dc942", "checked_on": "2026-10-18" }, { "document_id": "ds1017-runs-2026-10-17-PROVENANCE.md", "parent_record_id": "deepseek-1017-tensorfold-spark", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-17/PROVENANCE.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/PROVENANCE.md", "document_sha256": "aa020bd6aa0fe840c9518c96dbc00f04bbdf6add834f2fe7bac2b076aa91a089", "checked_on": "2026-10-18" }, { "document_id": "ds1017-runs-2026-10-18-INDEX.md", "parent_record_id": "deepseek-1017-tensorfold-spark", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-18/INDEX.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md", "document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "checked_on": "2026-10-18" }, { "document_id": "ds1017-runs-2026-10-17-01_vllm-miaai-kit_suite-results_32of32.json", "parent_record_id": "deepseek-1017-tensorfold-spark", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-17/01_vllm-miaai-kit_suite-results_32of32.json", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/01_vllm-miaai-kit_suite-results_32of32.json", "document_sha256": "0cb7c61b5b321ba0b9eec76460e4d9d3d0e759bd09cfb03fc747ef66e7c2ea63", "checked_on": "2026-10-18" }, { "document_id": "ds1017-runs-2026-10-17-11_tensorfold-fd2365042f56_suite-results_32of32.json", "parent_record_id": "deepseek-1017-tensorfold-spark", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-17/11_tensorfold-fd2365042f56_suite-results_32of32.json", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/11_tensorfold-fd2365042f56_suite-results_32of32.json", "document_sha256": "d0ec8c99b0208ac4ddc272379d3cd71c796b5662d09481d6ba1e98fbbf0fa41c", "checked_on": "2026-10-18" }, { "document_id": "official-deepseek-ai--DeepSeek-V4.1-Flash-1017", "parent_record_id": "deepseek-1017-tensorfold-spark", "publisher": "deepseek-ai", "repository": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "document_path": "README.md", "pinned_url": "https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017/blob/1bae0e1088993b9b82de08f59de5be5a34e42170/README.md", "document_sha256": "b72b796e4f4c739eca5d0ad360a7679b39118ca7eb64891f7151a527b292c42a", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "Apache-2.0", "scope": "Repository metadata only; model conversion, base weights, plugin and engine terms retain their own scope." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Local Spark Qwen draft, source-audited and edited by Run AI Here with Codex; no independent model execution.", "answer": "The selected 1017 results describe two complete serving configurations, not an isolated engine-only test. Keep the converted weights and separate Engram revision, distinguish October 17 decode from October 18 prefill, and retain missing image, cache and quality evidence.", "sections": [ { "heading": "Which 1017 artifact was measured?", "paragraphs": [ "The report uses an EXL3 2.9bpw conversion at afc7aa0aab98ee735c0f54e0a122019af1ac08f1 plus separate official 1017 Engram tables at 1bae0e1088993b9b82de08f59de5be5a34e42170. The author does not establish the measured conversion is the newest revision. This is a particular converted artifact, not a speed result for unconverted official weights." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/PROVENANCE.md" ] }, { "heading": "What changes between the two decode arms?", "paragraphs": [ "Both use one request on the same reported model view and Spark pair. The vLLM kit has three drafts and a 262144 context cap; TensorFold run 11 has five drafts and a 1048576 cap. Their batching, configured concurrency and template paths also differ. The selected two-prompt means are observations of complete configurations, not an isolated engine effect." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/BENCHMARKS.md" ] }, { "heading": "Why not use the headline median as six repeats?", "paragraphs": [ "The report explicitly says its six TensorFold suite runs used different builds. Run 11 identifies fd2365042f56, whereas the current recipe builds 92aa710b533d. Keep those versions apart. The baseline has only one run, and there is no measured multi-user vLLM baseline." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/BENCHMARKS.md" ] }, { "heading": "What does the later prefill cohort establish?", "paragraphs": [ "Five October 18 source rows use the final setup, random-word prompts and recorded zero cached tokens. The selected 32K and 128K medians retain all five row values and their observed ranges. These are separate from the older repeated-sentence suite, where cache hits could not be excluded; range is not a confidence interval." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md" ] }, { "heading": "Where are the limits?", "paragraphs": [ "Image input and general accuracy were not measured. Near-million-token needle retrieval does not prove general reasoning, decode speed or simultaneous long-session capacity. Runtime caches, Engram storage and per-node memory headroom need their own checks. Remaining slower states are acknowledged by the author." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/KNOWN-LIMITS.md" ] } ], "facts": [], "reported_results": [ { "label": "vllm-kit-run01 prose", "metric": "decode_rate", "value": 27, "unit": "tokens/s", "statistic": "one run; mean of two prompt-level decode rates; rounded by source", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "vllm-kit-run01", "engine": "vLLM + exllamav3 overlay", "engine_version": null, "runtime_execution_revision": null, "image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "kit_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "draft_tokens": 3, "configured_max_requests": 1, "context_limit_tokens": 262144, "prefill_chunk_tokens": 1536, "chat_template": "kit template; actual CHAT_TEMPLATE override not read by the source", "workload": "prose", "per_prompt_decode_rates": [ 24.7, 29.3 ], "per_prompt_TTFT_seconds": [ 0.32, 0.28 ], "prompt_count": 2 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/01_vllm-miaai-kit_suite-results_32of32.json#L6", "source_document_sha256": "0cb7c61b5b321ba0b9eec76460e4d9d3d0e759bd09cfb03fc747ef66e7c2ea63", "source_document_id": "ds1017-runs-2026-10-17-01_vllm-miaai-kit_suite-results_32of32.json", "source_line": 6 }, { "label": "vllm-kit-run01 code", "metric": "decode_rate", "value": 39.9, "unit": "tokens/s", "statistic": "one run; mean of two prompt-level decode rates; rounded by source", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "vllm-kit-run01", "engine": "vLLM + exllamav3 overlay", "engine_version": null, "runtime_execution_revision": null, "image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "kit_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "draft_tokens": 3, "configured_max_requests": 1, "context_limit_tokens": 262144, "prefill_chunk_tokens": 1536, "chat_template": "kit template; actual CHAT_TEMPLATE override not read by the source", "workload": "code", "per_prompt_decode_rates": [ 40.4, 39.4 ], "per_prompt_TTFT_seconds": [ 0.47, 0.32 ], "prompt_count": 2 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/01_vllm-miaai-kit_suite-results_32of32.json#L20", "source_document_sha256": "0cb7c61b5b321ba0b9eec76460e4d9d3d0e759bd09cfb03fc747ef66e7c2ea63", "source_document_id": "ds1017-runs-2026-10-17-01_vllm-miaai-kit_suite-results_32of32.json", "source_line": 20 }, { "label": "tensorfold-run11 prose", "metric": "decode_rate", "value": 56.9, "unit": "tokens/s", "statistic": "one run; mean of two prompt-level decode rates; rounded by source", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "tensorfold-run11", "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": "fd2365042f56fcf726cf4071132c8b779ad2eae0", "image_tag": "tensorfold-dsv41:fd2365042f56", "image_digest": null, "draft_tokens": 5, "configured_max_requests": 4, "context_limit_tokens": 1048576, "prefill_chunk_tokens": 2048, "chat_template": "patched kit template; source infers per-run use from file times", "workload": "prose", "per_prompt_decode_rates": [ 53.4, 60.4 ], "per_prompt_TTFT_seconds": [ 0.12, 0.12 ], "prompt_count": 2 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/11_tensorfold-fd2365042f56_suite-results_32of32.json#L6", "source_document_sha256": "d0ec8c99b0208ac4ddc272379d3cd71c796b5662d09481d6ba1e98fbbf0fa41c", "source_document_id": "ds1017-runs-2026-10-17-11_tensorfold-fd2365042f56_suite-results_32of32.json", "source_line": 6 }, { "label": "tensorfold-run11 code", "metric": "decode_rate", "value": 91.8, "unit": "tokens/s", "statistic": "one run; mean of two prompt-level decode rates; rounded by source", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-17", "concurrency": 1, "thinking": false, "temperature": 1, "top_p": 1, "max_output_tokens": 700, "output_tokens": null, "input_tokens": null, "prefix_cache": "present; cache-hit counts not logged in this suite", "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "(completion_tokens - 1) / (stream end - first chunk with content, reasoning or tool call)", "repetitions": 1, "checkpoint_is_latest_revision_verified": false, "id": "tensorfold-run11", "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": "fd2365042f56fcf726cf4071132c8b779ad2eae0", "image_tag": "tensorfold-dsv41:fd2365042f56", "image_digest": null, "draft_tokens": 5, "configured_max_requests": 4, "context_limit_tokens": 1048576, "prefill_chunk_tokens": 2048, "chat_template": "patched kit template; source infers per-run use from file times", "workload": "code", "per_prompt_decode_rates": [ 91.1, 92.5 ], "per_prompt_TTFT_seconds": [ 0.13, 0.16 ], "prompt_count": 2 }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-17/11_tensorfold-fd2365042f56_suite-results_32of32.json#L20", "source_document_sha256": "d0ec8c99b0208ac4ddc272379d3cd71c796b5662d09481d6ba1e98fbbf0fa41c", "source_document_id": "ds1017-runs-2026-10-17-11_tensorfold-fd2365042f56_suite-results_32of32.json", "source_line": 20 }, { "label": "TensorFold final setup 32K prefill", "metric": "prefill_rate", "value": 2165, "unit": "tokens/s", "statistic": "median of five source rows; individually recoverable in per_run_values", "uncertainty": { "observed_min": 2065, "observed_max": 2201, "interval_kind": "observed range, not confidence interval" }, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-18", "concurrency": 1, "thinking": false, "temperature": null, "top_p": null, "max_output_tokens": null, "output_tokens": null, "input_tokens": null, "prefix_cache": false, "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "prompt_tokens / TTFT; source final setup and host tuning", "repetitions": 5, "checkpoint_is_latest_revision_verified": false, "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": null, "runtime_revision_prefix": "92aa710b533d", "image_tag": "tensorfold-dsv41:92aa710b533d", "image_digest": null, "reported_prompt_bucket": "32K", "cached_tokens": 0, "per_run_values": [ 2201, 2180, 2065, 2165, 2100 ], "run_ids": [ "hw5", "hw6", "ht1", "ht2", "ht3" ], "draft_tokens": 5, "context_limit_tokens": 1048576, "warmup": "one 8K prompt; subsequent 8K requests before 32K/64K/128K; random-word prompts", "host_tuning_scope": "author applies no engine swap and memory-reclaim/compaction controls; not a universal host-tuning recommendation" }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L16", "source_document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "source_document_id": "ds1017-runs-2026-10-18-INDEX.md", "source_line": 16 }, { "label": "TensorFold final setup 128K prefill", "metric": "prefill_rate", "value": 1961, "unit": "tokens/s", "statistic": "median of five source rows; individually recoverable in per_run_values", "uncertainty": { "observed_min": 1851, "observed_max": 1986, "interval_kind": "observed range, not confidence interval" }, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "2 x NVIDIA DGX Spark GB10, TP2 over ConnectX-7", "node_count": 2, "driver": "580.178.04", "CUDA": "13.0", "experiment_date": "2026-10-18", "concurrency": 1, "thinking": false, "temperature": null, "top_p": null, "max_output_tokens": null, "output_tokens": null, "input_tokens": null, "prefix_cache": false, "kv_dtype": null, "kv_pool_tokens": null, "measurement_denominator": "prompt_tokens / TTFT; source final setup and host tuning", "repetitions": 5, "checkpoint_is_latest_revision_verified": false, "engine": "TensorFold", "engine_version": "0.6.6", "runtime_execution_revision": null, "runtime_revision_prefix": "92aa710b533d", "image_tag": "tensorfold-dsv41:92aa710b533d", "image_digest": null, "reported_prompt_bucket": "128K", "cached_tokens": 0, "per_run_values": [ 1986, 1981, 1914, 1961, 1851 ], "run_ids": [ "hw5", "hw6", "ht1", "ht2", "ht3" ], "draft_tokens": 5, "context_limit_tokens": 1048576, "warmup": "one 8K prompt; subsequent 8K requests before 32K/64K/128K; random-word prompts", "host_tuning_scope": "author applies no engine swap and memory-reclaim/compaction controls; not a universal host-tuning recommendation" }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L16", "source_document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "source_document_id": "ds1017-runs-2026-10-18-INDEX.md", "source_line": 16 } ], "source_document_sha256": "546a0c3bfbbb6e84226ec812cf38927f6d993e53b73633a540c426eea5f13d29" } } ] }, { "id": "qwen38-chisel", "title": "Read Qwen3.8 Chisel timing and negative quality gates", "needs": [ "catalog" ], "purpose": "Return the complete source report; preserve thinking modes, paired intervals and false non-inferiority flags.", "expression": "catalog.records.filter(r => r.id === 'qwen38-chisel-dflash-replay')", "expected_output": [ { "id": "qwen38-chisel-dflash-replay", "title": "Qwen3.8-27B: Chisel replay on one DGX Spark", "kind": "benchmark", "source_type": "community_operator_report", "summary": "A pinned single-Spark NVFP4/DFlash2 recipe with separate speed cohorts, task-level BF16 comparisons and negative non-inferiority flags.", "runtime": "vLLM", "runtime_topics": [ "vLLM" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "Qwen3.8-27B", "model_family": "Qwen", "model_id": "unsloth/Qwen3.8-27B-NVFP4", "model_release_date": null, "model_artifact": { "id": "unsloth/Qwen3.8-27B-NVFP4", "revision": null }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "1 x DGX Spark GB10; 128 GB unified memory; 2000 MHz cap in candidate timing cohort", "node_count": 1, "parallelism": "TP1", "kv_dtype": null, "context_limit_tokens": null, "experiment_date": "2026-10-15 source cohort; BF16 reference filed 2026-10-09" }, "test_profiles": [ { "label": "Chisel replay mixed-prompt median decode latency", "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L9", "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "thinking": true, "reasoning_effort": "medium", "greedy": true, "temperature": 0, "top_p": null, "concurrency": 1, "input_tokens": null, "max_output_tokens": 256, "output_tokens": null, "prompt_count": 25, "prompt_mix": { "chat": 9, "code": 9, "JSON_schema": 6, "long_story": 1 }, "repeats_per_session": 3, "fresh_start_sessions": 2, "measurement_denominator": "(last streamed token time - first) / (output tokens - 1); median latency, not whole-request served rate" } }, { "label": "Chisel replay mixed-prompt median TTFT", "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L10", "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "thinking": true, "reasoning_effort": "medium", "greedy": true, "temperature": 0, "top_p": null, "concurrency": 1, "input_tokens": null, "max_output_tokens": 256, "output_tokens": null, "prompt_count": 25, "prompt_mix": { "chat": 9, "code": 9, "JSON_schema": 6, "long_story": 1 }, "repeats_per_session": 3, "fresh_start_sessions": 2, "measurement_denominator": "request start to first streamed token" } }, { "label": "Local llama-benchy 16K C1", "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11", "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores", "concurrency": 1, "input_tokens": 2048, "retained_context_tokens": 16384, "output_tokens": 128, "prefix_cache": true, "repeats_per_session": 5, "fresh_start_sessions": 2, "thinking": null, "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2" } }, { "label": "Local llama-benchy 16K C2", "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11", "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores", "concurrency": 2, "input_tokens": 2048, "retained_context_tokens": 16384, "output_tokens": 128, "prefix_cache": true, "repeats_per_session": 5, "fresh_start_sessions": 2, "thinking": null, "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2" } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "inference_speed_and_task_checks", "notes": [ "The recipe is a concrete deployment candidate, but speed and quality use different request shapes and thinking modes. The source custom quality bar passes while its non-inferiority flags are false; preserve both and leave the executed local image digest unknown." ], "limitations": [ "Source-reported only; no independent site model runs.", "Document revision identifies the report, not the executed runtime or weight revision.", "Complete measured configurations and missing fields must accompany extracted values." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "yuanlin2004", "url": "https://github.com/yuanlin2004/chisel-spark-recipes", "repository": "yuanlin2004/chisel-spark-recipes", "platform": "github", "revision": "84feafddff1e834bd16b849c827eed11676e5934", "document_path": "recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md", "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md", "report_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md", "document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321", "last_commit_at": "2026-10-17T23:01:04Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.yaml", "parent_record_id": "qwen38-chisel-dflash-replay", "publisher": "yuanlin2004", "repository": "yuanlin2004/chisel-spark-recipes", "revision": "84feafddff1e834bd16b849c827eed11676e5934", "document_path": "recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.yaml", "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.yaml", "document_sha256": "028f2badf76407546be67b8bd93e2e81aae8ad5d00076e20b8bd8835021a1ede", "checked_on": "2026-10-18" }, { "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-ENVIRONMENT.md", "parent_record_id": "qwen38-chisel-dflash-replay", "publisher": "yuanlin2004", "repository": "yuanlin2004/chisel-spark-recipes", "revision": "84feafddff1e834bd16b849c827eed11676e5934", "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/ENVIRONMENT.md", "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/ENVIRONMENT.md", "document_sha256": "349c8de9b51a1df3ebba677db79d4e62c9a652a6dc225965f7515b80583d471a", "checked_on": "2026-10-18" }, { "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-METHOD.md", "parent_record_id": "qwen38-chisel-dflash-replay", "publisher": "yuanlin2004", "repository": "yuanlin2004/chisel-spark-recipes", "revision": "84feafddff1e834bd16b849c827eed11676e5934", "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/METHOD.md", "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/METHOD.md", "document_sha256": "072fb07c8e46bb6e1e7a4c8a9416b94a87863cfc60dfde60f127e9243363dd8e", "checked_on": "2026-10-18" }, { "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-server.image", "parent_record_id": "qwen38-chisel-dflash-replay", "publisher": "yuanlin2004", "repository": "yuanlin2004/chisel-spark-recipes", "revision": "84feafddff1e834bd16b849c827eed11676e5934", "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/server.image", "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/server.image", "document_sha256": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", "checked_on": "2026-10-18" }, { "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "parent_record_id": "qwen38-chisel-dflash-replay", "publisher": "yuanlin2004", "repository": "yuanlin2004/chisel-spark-recipes", "revision": "84feafddff1e834bd16b849c827eed11676e5934", "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json", "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json", "document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "checked_on": "2026-10-18" }, { "document_id": "chisel-tools-compare-quality", "parent_record_id": "qwen38-chisel-dflash-replay", "publisher": "yuanlin2004", "repository": "yuanlin2004/chisel-spark-recipes", "revision": "84feafddff1e834bd16b849c827eed11676e5934", "document_path": "tools/compare_quality.py", "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/tools/compare_quality.py", "document_sha256": "ee52180fce64fa6948d7516ac42f7f5fd0ef143584d2c284d6aa19f6fdecb783", "checked_on": "2026-10-18" }, { "document_id": "official-Qwen--Qwen3.8-27B", "parent_record_id": "qwen38-chisel-dflash-replay", "publisher": "Qwen", "repository": "Qwen/Qwen3.8-27B", "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "document_path": "README.md", "pinned_url": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md", "document_sha256": "57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "Apache-2.0", "scope": "Repository metadata only; model conversion, base weights, plugin and engine terms retain their own scope." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Local Spark Qwen draft, source-audited and edited by Run AI Here with Codex; no independent model execution.", "answer": "The recipe is a concrete deployment candidate, but speed and quality use different request shapes and thinking modes. The source custom quality bar passes while its non-inferiority flags are false; preserve both and leave the executed local image digest unknown.", "sections": [ { "heading": "What is pinned and what remains unknown?", "paragraphs": [ "The recipe names unsloth Qwen3.8-27B NVFP4, ten DFlash2 draft tokens, bf16 GDN state and chisel-passes 0.1.0. The measured target revision is only a short prefix. A recipe base-image digest is available, but the main tests used a local build and the saved server.image is empty; the executed image digest remains unknown." ], "citations": [ "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/ENVIRONMENT.md" ] }, { "heading": "Which speed cohort is being quoted?", "paragraphs": [ "The mixed-prompt test reports median decode latency and TTFT with thinking on and medium effort, one request at a time, up to 256 output tokens and two fresh-start sessions. Local llama-benchy results instead use 16K retained context, 128 output tokens and prefix cache on; its two-user throughput is aggregate, not per user." ], "citations": [ "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/METHOD.md" ] }, { "heading": "How should the quality gate be read?", "paragraphs": [ "Quality tests use thinking off and compare four tasks against a separately recorded BF16 reference. The custom bar passes, but all four non_inferior flags and the overall all_non_inferior flag are false. Preserve both gates, the raw scores, sample counts, paired differences and intervals. These results do not establish equivalence or general quality preservation." ], "citations": [ "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json" ] }, { "heading": "How were the intervals computed?", "paragraphs": [ "The comparison code uses matched per-question differences and a normal approximation: mean difference plus or minus 1.96 sample standard errors. It is not a bootstrap interval. The non-inferiority flag requires the lower bound above a loss of one point, while the author custom bar uses score recovery, a clearly-worse check and answer-length limits." ], "citations": [ "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/tools/compare_quality.py" ] }, { "heading": "What should an operator verify next?", "paragraphs": [ "The configured 262144 context cap and four-sequence capacity are launch settings. They do not establish simultaneous near-limit fit. Verify the exact quantized artifact, drafter, plugin and image, then use representative requests and correctness tasks. Do not transfer candidate-specific image, clock or plugin settings to the BF16 reference when they were not reported." ], "citations": [ "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.yaml" ] } ], "facts": [], "reported_results": [ { "label": "Chisel replay mixed-prompt median decode latency", "metric": "decode_latency", "value": 15.35, "unit": "ms/token", "statistic": "source-reported median over two fresh-start sessions", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "thinking": true, "reasoning_effort": "medium", "greedy": true, "temperature": 0, "top_p": null, "concurrency": 1, "input_tokens": null, "max_output_tokens": 256, "output_tokens": null, "prompt_count": 25, "prompt_mix": { "chat": 9, "code": 9, "JSON_schema": 6, "long_story": 1 }, "repeats_per_session": 3, "fresh_start_sessions": 2, "measurement_denominator": "(last streamed token time - first) / (output tokens - 1); median latency, not whole-request served rate" }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L9", "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321", "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md", "source_line": 9 }, { "label": "Chisel replay mixed-prompt median TTFT", "metric": "ttft", "value": 241, "unit": "ms", "statistic": "source-reported median on the same prompt cohort", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "thinking": true, "reasoning_effort": "medium", "greedy": true, "temperature": 0, "top_p": null, "concurrency": 1, "input_tokens": null, "max_output_tokens": 256, "output_tokens": null, "prompt_count": 25, "prompt_mix": { "chat": 9, "code": 9, "JSON_schema": 6, "long_story": 1 }, "repeats_per_session": 3, "fresh_start_sessions": 2, "measurement_denominator": "request start to first streamed token" }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L10", "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321", "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md", "source_line": 10 }, { "label": "Local llama-benchy 16K C1", "metric": "generation_throughput", "value": 30, "unit": "tokens/s", "statistic": "source-reported summary; separate from mixed-prompt median latency", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores", "concurrency": 1, "input_tokens": 2048, "retained_context_tokens": 16384, "output_tokens": 128, "prefix_cache": true, "repeats_per_session": 5, "fresh_start_sessions": 2, "thinking": null, "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2" }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11", "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321", "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md", "source_line": 11 }, { "label": "Local llama-benchy 16K C2", "metric": "generation_throughput", "value": 58, "unit": "tokens/s", "statistic": "source-reported summary; separate from mixed-prompt median latency", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight", "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores", "concurrency": 2, "input_tokens": 2048, "retained_context_tokens": 16384, "output_tokens": 128, "prefix_cache": true, "repeats_per_session": 5, "fresh_start_sessions": 2, "thinking": null, "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2" }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11", "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321", "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md", "source_line": 11 }, { "label": "gsm8k BF16 reference", "metric": "task_score", "value": 90.06823351023503, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "Qwen/Qwen3.8-27B", "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reported_model_revision_prefix": null, "system": null, "node_count": null, "driver": null, "GPU_clock_cap_MHz": null, "engine": null, "executed_engine_version": null, "runtime_execution_revision": null, "recipe_base_image_digest": null, "executed_local_image_digest": null, "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.", "plugin": null, "draft_model": null, "draft_weight_sha256": null, "draft_tokens": null, "GDN_state_dtype": null, "KV_dtype": null, "configured_context_limit_tokens": null, "configured_max_requests": null, "GPU_memory_utilization": null, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "BF16 reference", "task": "gsm8k", "n": 1319, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "5-shot accuracy", "paired_difference_normal_CI95_points": [ -1.8137451343135735, 0.6007049523575464 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L8", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 8 }, { "label": "gsm8k NVFP4 replay", "metric": "task_score", "value": 89.46171341925702, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "NVFP4 replay", "task": "gsm8k", "n": 1319, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "5-shot accuracy", "paired_difference_normal_CI95_points": [ -1.8137451343135735, 0.6007049523575464 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L9", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 9 }, { "label": "ifeval BF16 reference", "metric": "task_score", "value": 80.96118299445472, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "Qwen/Qwen3.8-27B", "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reported_model_revision_prefix": null, "system": null, "node_count": null, "driver": null, "GPU_clock_cap_MHz": null, "engine": null, "executed_engine_version": null, "runtime_execution_revision": null, "recipe_base_image_digest": null, "executed_local_image_digest": null, "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.", "plugin": null, "draft_model": null, "draft_weight_sha256": null, "draft_tokens": null, "GDN_state_dtype": null, "KV_dtype": null, "configured_context_limit_tokens": null, "configured_max_requests": null, "GPU_memory_utilization": null, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "BF16 reference", "task": "ifeval", "n": 541, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "prompt-level strict", "paired_difference_normal_CI95_points": [ -1.560611007417151, 3.409039842906985 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L26", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 26 }, { "label": "ifeval NVFP4 replay", "metric": "task_score", "value": 81.88539741219964, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "NVFP4 replay", "task": "ifeval", "n": 541, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "prompt-level strict", "paired_difference_normal_CI95_points": [ -1.560611007417151, 3.409039842906985 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L27", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 27 }, { "label": "humaneval_chat BF16 reference", "metric": "task_score", "value": 95.1219512195122, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "Qwen/Qwen3.8-27B", "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reported_model_revision_prefix": null, "system": null, "node_count": null, "driver": null, "GPU_clock_cap_MHz": null, "engine": null, "executed_engine_version": null, "runtime_execution_revision": null, "recipe_base_image_digest": null, "executed_local_image_digest": null, "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.", "plugin": null, "draft_model": null, "draft_weight_sha256": null, "draft_tokens": null, "GDN_state_dtype": null, "KV_dtype": null, "configured_context_limit_tokens": null, "configured_max_requests": null, "GPU_memory_utilization": null, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "BF16 reference", "task": "humaneval_chat", "n": 164, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "chat pass@1", "paired_difference_normal_CI95_points": [ -2.6839968720184646, 1.4644846768965132 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L44", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 44 }, { "label": "humaneval_chat NVFP4 replay", "metric": "task_score", "value": 94.51219512195121, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "NVFP4 replay", "task": "humaneval_chat", "n": 164, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "chat pass@1", "paired_difference_normal_CI95_points": [ -2.6839968720184646, 1.4644846768965132 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L45", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 45 }, { "label": "longbench_narrativeqa BF16 reference", "metric": "task_score", "value": 34.83234254995091, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "Qwen/Qwen3.8-27B", "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reported_model_revision_prefix": null, "system": null, "node_count": null, "driver": null, "GPU_clock_cap_MHz": null, "engine": null, "executed_engine_version": null, "runtime_execution_revision": null, "recipe_base_image_digest": null, "executed_local_image_digest": null, "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.", "plugin": null, "draft_model": null, "draft_weight_sha256": null, "draft_tokens": null, "GDN_state_dtype": null, "KV_dtype": null, "configured_context_limit_tokens": null, "configured_max_requests": null, "GPU_memory_utilization": null, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "BF16 reference", "task": "longbench_narrativeqa", "n": 200, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "F1, full stories", "paired_difference_normal_CI95_points": [ -1.6421295670363527, 2.4483306144283086 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L62", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 62 }, { "label": "longbench_narrativeqa NVFP4 replay", "metric": "task_score", "value": 35.235443073646884, "unit": "score points", "statistic": "source paired-comparison raw score; retain task-specific definition", "uncertainty": null, "conditions": { "model": "Qwen3.8-27B", "model_artifact": "unsloth/Qwen3.8-27B-NVFP4", "model_revision": null, "reported_model_revision_prefix": "f0b7c9e7", "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory", "node_count": 1, "driver": "580.178.04", "GPU_clock_cap_MHz": 2000, "engine": "vLLM", "executed_engine_version": "0.31.1rc1.dev380", "runtime_execution_revision": null, "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0", "executed_local_image_digest": null, "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest", "plugin": "chisel-passes 0.1.0", "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter", "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c", "draft_tokens": 10, "GDN_state_dtype": "bfloat16", "KV_dtype": null, "configured_context_limit_tokens": 262144, "configured_max_requests": 4, "GPU_memory_utilization": 0.6, "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths", "quality_arm": "NVFP4 replay", "task": "longbench_narrativeqa", "n": 200, "thinking": false, "greedy": true, "concurrency": null, "input_tokens": null, "output_tokens": null, "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task", "reference_model": "Qwen/Qwen3.8-27B", "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", "reference_weight_precision": "bfloat16", "reference_KV_dtype": "bfloat16", "metric_definition": "F1, full stories", "paired_difference_normal_CI95_points": [ -1.6421295670363527, 2.4483306144283086 ], "non_inferior": false, "author_custom_bar_passed": true }, "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L63", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 63 } ], "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321" }, "quality_comparisons": [ { "task": "gsm8k", "n": 1319, "reference_score": 90.06823351023503, "candidate_score": 89.46171341925702, "diff_points": -0.6065200909780136, "ci95_points": [ -1.8137451343135735, 0.6007049523575464 ], "non_inferior": false, "length_ratio": 1.0086485431409682, "flips_right_to_wrong": 37, "flips_wrong_to_right": 29, "recovery": 0.9932659932659933, "clearly_worse": false, "length_ok": true, "passes": true, "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap", "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L6", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 6 }, { "task": "ifeval", "n": 541, "reference_score": 80.96118299445472, "candidate_score": 81.88539741219964, "diff_points": 0.9242144177449169, "ci95_points": [ -1.560611007417151, 3.409039842906985 ], "non_inferior": false, "length_ratio": 1.001384198457369, "flips_right_to_wrong": 21, "flips_wrong_to_right": 26, "recovery": 1.0114155251141552, "clearly_worse": false, "length_ok": true, "passes": true, "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap", "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L24", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 24 }, { "task": "humaneval_chat", "n": 164, "reference_score": 95.1219512195122, "candidate_score": 94.51219512195121, "diff_points": -0.6097560975609756, "ci95_points": [ -2.6839968720184646, 1.4644846768965132 ], "non_inferior": false, "length_ratio": 1.0066489055410106, "flips_right_to_wrong": 2, "flips_wrong_to_right": 1, "recovery": 0.9935897435897434, "clearly_worse": false, "length_ok": true, "passes": true, "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap", "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L42", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 42 }, { "task": "longbench_narrativeqa", "n": 200, "reference_score": 34.83234254995091, "candidate_score": 35.235443073646884, "diff_points": 0.403100523695978, "ci95_points": [ -1.6421295670363527, 2.4483306144283086 ], "non_inferior": false, "length_ratio": 0.9618899475014583, "recovery": 1.011572593003698, "clearly_worse": false, "length_ok": true, "passes": true, "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap", "evidence_state": "source_reported", "site_reproduced": false, "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L60", "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb", "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json", "source_line": 60 } ] } ] }, { "id": "qwen-flash-next-memory", "title": "Read Qwen Flash-Next memory types and cold-context conditions", "needs": [ "catalog" ], "purpose": "Return both complete reports; keep Spark and RTX PRO hardware, mmap/anonymous memory and cold-prefix cohorts separate.", "expression": "catalog.records.filter(r => ['qwen38-flash-next-single-spark-context','qwen38-flash-next-ple-memory'].includes(r.id))", "expected_output": [ { "id": "qwen38-flash-next-single-spark-context", "title": "Qwen Flash-Next: single-Spark memory and cold context", "kind": "benchmark", "source_type": "community_operator_report", "summary": "The v2.4.0 promotion arm serves the pinned GDN-MSE checkpoint, with an extra MXFP8 prefill shard. NVFP4 expert weights, MXFP8 paths and a refitted four-draft MTP head are separate components. The experimental image tag is retained; the later release-image digest does not prove the exact cold-test image digest. Repository release names are specific to this one-Spark recipe.", "runtime": "vLLM", "runtime_topics": [ "vLLM" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "Qwen3.8-Flash-Next", "model_family": "Qwen", "model_id": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_release_date": null, "model_artifact": { "id": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "revision": "03f4a0570496bbe741189f4a2a7d611679e85406" }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "kv_dtype": "fp8", "context_limit_tokens": 262144, "experiment_date": "2026-10-16" }, "test_profiles": [ { "label": "1x v2.4.0 16384-token ttft", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1262", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 16384, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "ttft_s", "workload": "long-prompt" } }, { "label": "1x v2.4.0 16384-token prefill_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1263", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 16384, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "pp", "workload": "long-prompt" } }, { "label": "1x v2.4.0 65536-token ttft", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1264", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 65536, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "ttft_s", "workload": "long-prompt" } }, { "label": "1x v2.4.0 65536-token prefill_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1265", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 65536, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "pp", "workload": "long-prompt" } }, { "label": "1x v2.4.0 131072-token ttft", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1266", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 131072, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "ttft_s", "workload": "long-prompt" } }, { "label": "1x v2.4.0 131072-token prefill_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1267", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 131072, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "pp", "workload": "long-prompt" } }, { "label": "1x v2.4.0 allocated KV pool", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/BENCHMARKS.md#L106", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": null, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": null, "thinking": null, "output_tokens": null, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": null, "memory_kind": "allocated_KV_token_pool_not_measured_peak_RAM", "max_context_simultaneously_validated": false } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "source_memory_and_endpoint_observations", "notes": [ "A source document, not an independent experiment count. Preserve measurement type and unknown fields." ], "limitations": [ "No independent Run AI Here model execution.", "Document revision is distinct from execution and weight identity.", "File bytes, host RSS, anonymous memory, MemAvailable, KV allocation and configured context are different quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "ursuciprian", "url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark", "repository": "ursuciprian/qwen3.8-flash-next-1x-dgx-spark", "platform": "github", "revision": "348ede7e28a01f6aa6b501aaf2392d20a286a012", "document_path": "README.md", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/README.md", "report_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/README.md", "document_sha256": "bb618a879867f416c51cfc3fa8b699da427d28ec324b14af7d1279a2501ab37f", "last_commit_at": "2026-10-16T16:43:24Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "qwen1x-docs-data-capability.csv", "parent_record_id": "qwen38-flash-next-single-spark-context", "publisher": "ursuciprian", "repository": "ursuciprian/qwen3.8-flash-next-1x-dgx-spark", "revision": "348ede7e28a01f6aa6b501aaf2392d20a286a012", "document_path": "docs/data/capability.csv", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv", "document_sha256": "4376de2ea68ca23d5208731ce948a28801cd1da2dc026c9415b2b122e9339205", "checked_on": "2026-10-18" }, { "document_id": "qwen1x-docs-BENCHMARKS.md", "parent_record_id": "qwen38-flash-next-single-spark-context", "publisher": "ursuciprian", "repository": "ursuciprian/qwen3.8-flash-next-1x-dgx-spark", "revision": "348ede7e28a01f6aa6b501aaf2392d20a286a012", "document_path": "docs/BENCHMARKS.md", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/BENCHMARKS.md", "document_sha256": "ac2128626b607b2e4571bdb45fb7e2639e5c65c83841a8f28367bb5692986fc0", "checked_on": "2026-10-18" }, { "document_id": "qwen1x-VERSIONS.md", "parent_record_id": "qwen38-flash-next-single-spark-context", "publisher": "ursuciprian", "repository": "ursuciprian/qwen3.8-flash-next-1x-dgx-spark", "revision": "348ede7e28a01f6aa6b501aaf2392d20a286a012", "document_path": "VERSIONS.md", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/VERSIONS.md", "document_sha256": "c513523883d95b716707697a0633fb9c5e63c9cf748aeaf37a38d4f771913abd", "checked_on": "2026-10-18" }, { "document_id": "qwen1x-results-k103-1x-prefill-combo-ab-20261016-1206-cb-01-arm-p1-recipe.yaml", "parent_record_id": "qwen38-flash-next-single-spark-context", "publisher": "ursuciprian", "repository": "ursuciprian/qwen3.8-flash-next-1x-dgx-spark", "revision": "348ede7e28a01f6aa6b501aaf2392d20a286a012", "document_path": "results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/arm-p1/recipe.yaml", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/arm-p1/recipe.yaml", "document_sha256": "33ecbeea74e85987bd4dfa7418543438e98ff93ea2270c103deccab44a43fa7e", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "apache-2.0", "scope": "Source declaration only; base weights, converters, runtimes and source integration material retain their own terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-grounded editorial synthesis with local Spark Qwen assistance and Codex review; no site model benchmark.", "answer": "The v2.4.0 promotion arm serves the pinned GDN-MSE checkpoint, with an extra MXFP8 prefill shard. NVFP4 expert weights, MXFP8 paths and a refitted four-draft MTP head are separate components. The experimental image tag is retained; the later release-image digest does not prove the exact cold-test image digest. Repository release names are specific to this one-Spark recipe.", "sections": [ { "heading": "Which quantization and build were tested?", "paragraphs": [ "The v2.4.0 promotion arm serves the pinned GDN-MSE checkpoint, with an extra MXFP8 prefill shard. NVFP4 expert weights, MXFP8 paths and a refitted four-draft MTP head are separate components. The experimental image tag is retained; the later release-image digest does not prove the exact cold-test image digest. Repository release names are specific to this one-Spark recipe." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/VERSIONS.md", "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/arm-p1/recipe.yaml" ] }, { "heading": "What does the long-prompt curve measure?", "paragraphs": [ "The selected cold-prefix observations cover 16384, 65536 and 131072 input tokens at C1, temperature zero and one requested output token. Values are means of four boots with three cold prompts each; reported SD is between boot means. A cold prefix does not establish cold operating-system page cache. These are prefill/TTFT observations, not long-response decode or broad reasoning accuracy." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv", "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/BENCHMARKS.md" ] }, { "heading": "How should memory limits be read?", "paragraphs": [ "The recipe allocates a 14 GiB FP8 KV pool and reports 993754 token capacity, while its request window is 262144 and scheduler ceiling is eight requests. The token pool is shared: these numbers do not demonstrate eight simultaneous full windows. PLE uses NVMe mmap and page cache. The 0.01 GPU-memory-utilization setting is a startup guard with explicit KV bytes, not measured one-percent memory use." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/README.md", "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/arm-p1/recipe.yaml" ] }, { "heading": "How should two-Spark comparisons be selected?", "paragraphs": [ "The project numbers releases independently for one-Spark and two-Spark setups. Its capability table also combines dated measurements when a newer build has no corresponding cell. Keep setup, release, workload and original run path together instead of treating every headline cell as one matched scaling experiment." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/VERSIONS.md" ] } ], "facts": [], "reported_results": [ { "label": "1x v2.4.0 16384-token ttft", "metric": "ttft", "value": 7.104, "unit": "s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "kind": "SD between boot means, not confidence interval", "standard_deviation": 0.016 }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 16384, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "ttft_s", "workload": "long-prompt" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen1x-docs-data-capability.csv", "source_document_sha256": "4376de2ea68ca23d5208731ce948a28801cd1da2dc026c9415b2b122e9339205", "source_line": 1262, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1262" }, { "label": "1x v2.4.0 16384-token prefill_rate", "metric": "prefill_rate", "value": 2306.4, "unit": "tokens/s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "kind": "SD between boot means, not confidence interval", "standard_deviation": 5.2 }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 16384, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "pp", "workload": "long-prompt" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen1x-docs-data-capability.csv", "source_document_sha256": "4376de2ea68ca23d5208731ce948a28801cd1da2dc026c9415b2b122e9339205", "source_line": 1263, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1263" }, { "label": "1x v2.4.0 65536-token ttft", "metric": "ttft", "value": 30.723, "unit": "s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "kind": "SD between boot means, not confidence interval", "standard_deviation": 0.06 }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 65536, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "ttft_s", "workload": "long-prompt" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen1x-docs-data-capability.csv", "source_document_sha256": "4376de2ea68ca23d5208731ce948a28801cd1da2dc026c9415b2b122e9339205", "source_line": 1264, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1264" }, { "label": "1x v2.4.0 65536-token prefill_rate", "metric": "prefill_rate", "value": 2133.2, "unit": "tokens/s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "kind": "SD between boot means, not confidence interval", "standard_deviation": 4.2 }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 65536, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "pp", "workload": "long-prompt" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen1x-docs-data-capability.csv", "source_document_sha256": "4376de2ea68ca23d5208731ce948a28801cd1da2dc026c9415b2b122e9339205", "source_line": 1265, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1265" }, { "label": "1x v2.4.0 131072-token ttft", "metric": "ttft", "value": 70, "unit": "s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "kind": "SD between boot means, not confidence interval", "standard_deviation": 0.137 }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 131072, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "ttft_s", "workload": "long-prompt" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen1x-docs-data-capability.csv", "source_document_sha256": "4376de2ea68ca23d5208731ce948a28801cd1da2dc026c9415b2b122e9339205", "source_line": 1266, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1266" }, { "label": "1x v2.4.0 131072-token prefill_rate", "metric": "prefill_rate", "value": 1872.5, "unit": "tokens/s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "kind": "SD between boot means, not confidence interval", "standard_deviation": 3.7 }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": 1, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": 0, "thinking": null, "output_tokens": 1, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": 131072, "repetitions": "4 boots x 3 cold prompts; SD between boots", "harness": "cold.py (k101)", "source_run_directory": "1x:results/k103-1x-prefill-combo-ab-20261016-1206/cb-01/", "source_metric": "pp", "workload": "long-prompt" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen1x-docs-data-capability.csv", "source_document_sha256": "4376de2ea68ca23d5208731ce948a28801cd1da2dc026c9415b2b122e9339205", "source_line": 1267, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/data/capability.csv#L1267" }, { "label": "1x v2.4.0 allocated KV pool", "metric": "kv_pool_tokens", "value": 993754, "unit": "tokens", "statistic": "boot allocation observed on every reported boot", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_artifact": "local-inference-lab/Qwen3.8-Flash-Next-NVFP4", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "weight_precision": "NVFP4 experts; MXFP8 dense/attention; GDN NVFP4 decode and MXFP8 prefill", "system": "one DGX Spark GB10 with 128 GB unified memory, local NVMe", "node_count": 1, "parallelism": "TP1", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "runtime_revision_prefix": "10d0acc8 files over 5dad364d", "b12x_revision_prefix": "a1ad1b69", "executed_image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "release_image_is_exact_cold_test_image": false, "recipe_release": "1x v2.4.0 promotion A/B; k103 cb test arm, not the release-image ship check", "experiment_date": "2026-10-16", "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_tokens": 4, "kv_dtype": "fp8", "GPU_memory_utilization_startup_guard": 0.01, "configured_KV_pool_bytes": 15032385536, "PLE_backing": "NVMe mmap, page cache; reported 26.8 GiB PLE table", "concurrency": null, "cache_state": "cold prefix, no cache hit; not a claimed cold OS page cache", "temperature": null, "thinking": null, "output_tokens": null, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "input_tokens": null, "memory_kind": "allocated_KV_token_pool_not_measured_peak_RAM", "max_context_simultaneously_validated": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen1x-docs-BENCHMARKS.md", "source_document_sha256": "ac2128626b607b2e4571bdb45fb7e2639e5c65c83841a8f28367bb5692986fc0", "source_line": 106, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-1x-dgx-spark/blob/348ede7e28a01f6aa6b501aaf2392d20a286a012/docs/BENCHMARKS.md#L106" } ], "source_document_sha256": "bb618a879867f416c51cfc3fa8b699da427d28ec324b14af7d1279a2501ab37f" }, "memory_observations": [ { "id": "qwen-1x-v2.4-cold", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "checkpoint_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "engine": "vLLM + b12x", "image_tag": "ghcr.io/ursuciprian/spark-vllm-b12x:k103-20261015-a1ad1b69-10d0acc8", "executed_image_digest": null, "release_image_digest": "sha256:30fe678ed7ae5d4deb277be137497bcf597e46e829471b7bed24edd4294a84f1", "context_limit_tokens": 262144, "measured_cold_input_tokens": [ 16384, 65536, 131072 ], "configured_max_requests": 8, "KV_pool_bytes": 15032385536, "KV_pool_tokens": 993754, "MTP_draft_tokens": 4, "PLE_file_reported_GiB": 26.8, "all_eight_full_window_requests_proven": false, "cold_prefix_is_not_cold_page_cache": true } ] }, { "id": "qwen38-flash-next-ple-memory", "title": "Qwen Flash-Next: quantized PLE and memory residence", "kind": "benchmark", "source_type": "community_operator_report", "summary": "This experiment changes the n-gram PLE table backing, rather than quantizing every model component. The source compares a BF16 table held in RAM with file-mapped sidecars on one 96 GB RTX PRO 6000 and a 176 GB host. Main-table throughput uses 8K input, 512 requested output tokens and no prefix-cache hits. Full checkpoint revision and executed image digest are not supplied.", "runtime": "vLLM", "runtime_topics": [ "vLLM" ], "hardware_topics": [ "nvidia" ], "platform_topics": [ "linux" ], "model_label": "Qwen3.8-Flash-Next", "model_family": "Qwen", "model_id": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_release_date": null, "model_artifact": { "id": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "revision": null }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "parallelism": null, "kv_dtype": null, "context_limit_tokens": 32768, "experiment_date": null }, "test_profiles": [ { "label": "BF16 in RAM table size", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L96", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "BF16 in RAM", "memory_kind": "PLE_artifact_size", "memory_backing": "in RAM" } }, { "label": "BF16 in RAM host RSS", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L96", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "BF16 in RAM", "memory_kind": "host_RSS_not_necessarily_anonymous", "memory_backing": "in RAM" } }, { "label": "INT4 group-16 mmap table size", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L98", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16 mmap", "memory_kind": "PLE_artifact_size", "memory_backing": "reclaimable mapped file pages" } }, { "label": "INT4 group-16 mmap host RSS", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L98", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16 mmap", "memory_kind": "host_RSS_not_necessarily_anonymous", "memory_backing": "reclaimable mapped file pages" } }, { "label": "mmap cold first request", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L68", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": 1, "cache_state": "page cache dropped before each arm", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 0, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": null, "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "workload": "first request, exact request shape unreported; optional resident-vs-mmap cohort, not main 8K/512 table", "memory_backing": "NVMe mmap" } }, { "label": "resident cold first request", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L69", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": 1, "cache_state": "page cache dropped before each arm", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 0, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": null, "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "workload": "first request, exact request shape unreported; optional resident-vs-mmap cohort, not main 8K/512 table", "memory_backing": "anonymous resident copy" } }, { "label": "INT4 resident sidecar anonymous allocation", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L81", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": 36, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": null, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "memory_kind": "anonymous_resident_sidecar_not_total_RSS" } }, { "label": "optional residence cohort KV pool, MTP 0", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L80", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": 36, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": null, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 0, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "memory_kind": "allocated_KV_token_pool_not_measured_RAM" } }, { "label": "optional residence cohort KV pool, MTP 3", "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L80", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": 36, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": null, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 3, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "memory_kind": "allocated_KV_token_pool_not_measured_RAM" } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "source_memory_and_endpoint_observations", "notes": [ "A source document, not an independent experiment count. Preserve measurement type and unknown fields." ], "limitations": [ "No independent Run AI Here model execution.", "Document revision is distinct from execution and weight identity.", "File bytes, host RSS, anonymous memory, MemAvailable, KV allocation and configured context are different quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "primitive-ai", "url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "repository": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "platform": "huggingface", "revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "document_path": "README.md", "pinned_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md", "report_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md", "document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "last_commit_at": "2026-09-14T15:58:31.000Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "qwenple-ples_int4-META.json", "parent_record_id": "qwen38-flash-next-ple-memory", "publisher": "primitive-ai", "repository": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "document_path": "ples_int4/META.json", "pinned_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/ples_int4/META.json", "document_sha256": "e9a1863008450b7ac329d50a3b6b908f21122b8bf5da999bdeff05004161884a", "checked_on": "2026-10-18" }, { "document_id": "qwenple-ples_fp8-META.json", "parent_record_id": "qwen38-flash-next-ple-memory", "publisher": "primitive-ai", "repository": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "document_path": "ples_fp8/META.json", "pinned_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/ples_fp8/META.json", "document_sha256": "de757efa51030ac1e4edb6acdb29507521f8ace860a3be511661a26e5017b632", "checked_on": "2026-10-18" }, { "document_id": "qwenple-ples_nvfp4-META.json", "parent_record_id": "qwen38-flash-next-ple-memory", "publisher": "primitive-ai", "repository": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "document_path": "ples_nvfp4/META.json", "pinned_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/ples_nvfp4/META.json", "document_sha256": "5aa4d53703f67c8c664d674c95f16c29c8ce7a94a2d7d9ac27ae8afe4c0b56ef", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "apache-2.0", "scope": "Source declaration only; base weights, converters, runtimes and source integration material retain their own terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-grounded editorial synthesis with local Spark Qwen assistance and Codex review; no site model benchmark.", "answer": "This experiment changes the n-gram PLE table backing, rather than quantizing every model component. The source compares a BF16 table held in RAM with file-mapped sidecars on one 96 GB RTX PRO 6000 and a 176 GB host. Main-table throughput uses 8K input, 512 requested output tokens and no prefix-cache hits. Full checkpoint revision and executed image digest are not supplied.", "sections": [ { "heading": "Which allocation is reduced?", "paragraphs": [ "This experiment changes the n-gram PLE table backing, rather than quantizing every model component. The source compares a BF16 table held in RAM with file-mapped sidecars on one 96 GB RTX PRO 6000 and a 176 GB host. Main-table throughput uses 8K input, 512 requested output tokens and no prefix-cache hits. Full checkpoint revision and executed image digest are not supplied." ], "citations": [ "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md", "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/ples_int4/META.json" ] }, { "heading": "Why is RSS different from anonymous memory?", "paragraphs": [ "The selected BF16 and INT4 rows retain artifact size separately from host RSS. File-mapped RSS includes reclaimable page-cache pages; it is not a permanent anonymous allocation. The optional INT4 resident copy instead adds a reported 29.8 GiB of anonymous sidecar memory, above the worker allocation. Source-labelled GB and GiB are not silently equated." ], "citations": [ "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md" ] }, { "heading": "What remains outside the evidence?", "paragraphs": [ "The optional cold-residence cohort drops page cache before each arm and has a different, incompletely reported request shape. Its first-request speed is not a warm throughput average. MTP changes the allocated KV token pool; mmap versus resident does not change that pool within each reported MTP setting. Neither the table-size reduction nor suite-level sanity proves general quality equivalence." ], "citations": [ "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md" ] }, { "heading": "Does the native window fit every launch configuration?", "paragraphs": [ "The source reports a native-window startup failure with MTP when 7.57 GiB of KV was required but 5.95 GiB remained. Its selected main table used a shorter 32768-token launch window, while a separate field report used 135168. These are configuration-specific observations, not a universal maximum or evidence that every format fits the native window." ], "citations": [ "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md" ] } ], "facts": [], "reported_results": [ { "label": "BF16 in RAM table size", "metric": "artifact_size", "value": 95.4, "unit": "GB", "statistic": "source-reported PLE table size, not whole model or peak RAM", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "BF16 in RAM", "memory_kind": "PLE_artifact_size", "memory_backing": "in RAM" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 96, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L96" }, { "label": "BF16 in RAM host RSS", "metric": "host_rss", "value": 95, "unit": "GB", "statistic": "approximate source host RSS", "uncertainty": { "approximate": true }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "BF16 in RAM", "memory_kind": "host_RSS_not_necessarily_anonymous", "memory_backing": "in RAM" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 96, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L96" }, { "label": "INT4 group-16 mmap table size", "metric": "artifact_size", "value": 32, "unit": "GB", "statistic": "source-reported PLE table size, not whole model or peak RAM", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16 mmap", "memory_kind": "PLE_artifact_size", "memory_backing": "reclaimable mapped file pages" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 98, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L98" }, { "label": "INT4 group-16 mmap host RSS", "metric": "host_rss", "value": 32.9, "unit": "GB", "statistic": "source host RSS including reclaimable file pages", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": 32768, "configured_max_requests": 36, "input_tokens": 8192, "output_tokens": 512, "actual_output_tokens": null, "concurrency": 1, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": true, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16 mmap", "memory_kind": "host_RSS_not_necessarily_anonymous", "memory_backing": "reclaimable mapped file pages" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 98, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L98" }, { "label": "mmap cold first request", "metric": "decode_rate", "value": 19, "unit": "tokens/s", "statistic": "first-request observation; warm asymptote is a separate quantity", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": 1, "cache_state": "page cache dropped before each arm", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 0, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": null, "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "workload": "first request, exact request shape unreported; optional resident-vs-mmap cohort, not main 8K/512 table", "memory_backing": "NVMe mmap" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 68, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L68" }, { "label": "resident cold first request", "metric": "decode_rate", "value": 74.3, "unit": "tokens/s", "statistic": "first-request observation; warm asymptote is a separate quantity", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": 1, "cache_state": "page cache dropped before each arm", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 0, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": null, "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "workload": "first request, exact request shape unreported; optional resident-vs-mmap cohort, not main 8K/512 table", "memory_backing": "anonymous resident copy" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 69, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L69" }, { "label": "INT4 resident sidecar anonymous allocation", "metric": "anonymous_memory", "value": 29.8, "unit": "GiB", "statistic": "sidecar allocation only, on top of roughly 8 GiB held by worker", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": 36, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": null, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": null, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "memory_kind": "anonymous_resident_sidecar_not_total_RSS" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 81, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L81" }, { "label": "optional residence cohort KV pool, MTP 0", "metric": "kv_pool_tokens", "value": 426639, "unit": "tokens", "statistic": "reported identical mmap/resident allocation for this MTP setting", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": 36, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": null, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 0, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "memory_kind": "allocated_KV_token_pool_not_measured_RAM" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 80, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L80" }, { "label": "optional residence cohort KV pool, MTP 3", "metric": "kv_pool_tokens", "value": 134192, "unit": "tokens", "statistic": "reported identical mmap/resident allocation for this MTP setting", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "primitive-ai/Qwen3.8-Flash-Next-mixed-NVFP4-FP8", "model_revision": null, "PLE_artifact": "primitive-ai/Qwen3.8-Flash-Next-PLE-quant", "PLE_artifact_revision": "a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763", "system": "one RTX PRO 6000 Blackwell 96 GB, 176 GB host RAM, local NVMe", "node_count": 1, "engine": "vLLM with exact-image PLE overlay", "engine_version": null, "runtime_execution_revision": null, "image_tag": "vllm/vllm-openai:qwen38-flash-next", "executed_image_digest": null, "experiment_date": null, "source_update_boundary": "2026-09-06 corrections present; individual table trial dates not stated", "model_weight_precision": "mixed NVFP4/FP8 checkpoint; PLE format varies", "context_limit_tokens": null, "configured_max_requests": 36, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "concurrency": null, "cache_state": "prefix-cache-free; mmap page-cache residency not independently logged", "thinking": null, "temperature": null, "top_p": null, "draft_tokens": 3, "MTP_main_table_execution": null, "kv_dtype": null, "throughput_repetitions": "two seeds, a/b; not independent boots", "memory_sampling_instant": null, "reported_size_unit": "GB as labelled by source, byte convention not independently specified", "PLE_format": "INT4 group-16", "memory_kind": "allocated_KV_token_pool_not_measured_RAM" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwenple-readme", "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9", "source_line": 80, "evidence_url": "https://huggingface.co/primitive-ai/Qwen3.8-Flash-Next-PLE-quant/blob/a0fa93f2b9ed5fcab0cf01b60e88ff1ce13a4763/README.md#L80" } ], "source_document_sha256": "0085c022b212b241827a09f1a315170737f76298ca5af3bd8a46dea2032faab9" }, "memory_observations": [ { "record_id": "qwen38-flash-next-ple-memory", "kind": "source_main_table_host_RSS", "BF16_in_RAM_GB_approx": 95, "INT4_mmap_GB": 32.9, "mapped_pages_reclaimable": true, "anonymous_memory_inferred_from_RSS": false, "GB_byte_convention_verified": false }, { "record_id": "qwen38-flash-next-ple-memory", "kind": "optional_anonymous_residence", "INT4_sidecar_GiB": 29.8, "additional_worker_GiB_approx": 8, "same_cohort_as_main_table": false } ] } ] }, { "id": "deepseek-1017-long-context", "title": "Read near-1M 1017 needle cases with output and headroom", "needs": [ "catalog" ], "purpose": "Retain both saved prompts, seven-token completions, same-depth retrieval scope, unknown sampling and minimum MemAvailable.", "expression": "catalog.records.filter(r => r.id === 'deepseek-1017-million-token-needle')", "expected_output": [ { "id": "deepseek-1017-million-token-needle", "title": "DeepSeek 1017: near-million-token needle and memory floor", "kind": "benchmark", "source_type": "community_operator_report", "summary": "Two saved runs found one exact-string needle at 63 percent depth in random filler, with 1034753 and 1035020 prompt tokens. Each completion is seven tokens, with zero cached prompt tokens. These are two different build/setup observations, not repeated measurements of one unchanged configuration or evidence of general long-document reasoning.", "runtime": "TensorFold", "runtime_topics": [ "TensorFold" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "DeepSeek-V4.1-Flash-1017", "model_family": "DeepSeek", "model_id": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_release_date": null, "model_artifact": { "id": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1" }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "two DGX Spark GB10, TP2", "node_count": 2, "parallelism": null, "kv_dtype": null, "context_limit_tokens": 1048576, "experiment_date": "2026-10-18T01:51:54Z" }, "test_profiles": [ { "label": "fd2365042f56 near-1M first token", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json#L8", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "fd2365042f56", "input_tokens": 1034753, "output_tokens": 7, "cached_tokens": 0, "cache_state": "zero prefix-cache hit tokens; not a cold OS page-cache guarantee", "experiment_date": "2026-10-18T01:51:54Z", "needle_found": true, "reasoning_tokens": 0, "total_seconds": 835.8 } }, { "label": "92aa710b533d near-1M first token", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json#L8", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cached_tokens": 0, "cache_state": "zero prefix-cache hit tokens; not a cold OS page-cache guarantee", "experiment_date": "2026-10-18T04:47:07Z", "needle_found": true, "reasoning_tokens": 0, "total_seconds": 825.9 } }, { "label": "near-1M final head minimum MemAvailable", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L59", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cache_state": "same final needle; cached_tokens 0", "experiment_date": "2026-10-18", "memory_kind": "minimum_host_MemAvailable_during_prefill", "node": "head", "earlyoom_limit_MiB_at_measurement": 1536, "earlyoom_later_limit_MiB": 1024, "OS_memory_controls_changed": true, "ordinary_host_baseline_inferred": false } }, { "label": "near-1M final worker minimum MemAvailable", "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L59", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cache_state": "same final needle; cached_tokens 0", "experiment_date": "2026-10-18", "memory_kind": "minimum_host_MemAvailable_during_prefill", "node": "worker", "earlyoom_limit_MiB_at_measurement": 1536, "earlyoom_later_limit_MiB": 1024, "OS_memory_controls_changed": true, "ordinary_host_baseline_inferred": false } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "source_memory_and_endpoint_observations", "notes": [ "A source document, not an independent experiment count. Preserve measurement type and unknown fields." ], "limitations": [ "No independent Run AI Here model execution.", "Document revision is distinct from execution and weight identity.", "File bytes, host RSS, anonymous memory, MemAvailable, KV allocation and configured context are different quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "Elevatormusic", "url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "platform": "github", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-18/INDEX.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md", "report_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md", "document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "last_commit_at": "2026-10-18T05:40:29Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "dsneedle-runs-2026-10-18-needle-1m.json", "parent_record_id": "deepseek-1017-million-token-needle", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-18/needle-1m.json", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json", "document_sha256": "9b81971c1378ab5b0bef466219c3a7aa7e2f78315bddbfc5bd0837cfec2fde40", "checked_on": "2026-10-18" }, { "document_id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "parent_record_id": "deepseek-1017-million-token-needle", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-18/needle-1m-final.json", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json", "document_sha256": "c476006b019d46c68a334ad3763f3e8e0f0532d01dfc6fb800c043c225709ab1", "checked_on": "2026-10-18" }, { "document_id": "dsneedle-docs-KNOWN-LIMITS.md", "parent_record_id": "deepseek-1017-million-token-needle", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "docs/KNOWN-LIMITS.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/KNOWN-LIMITS.md", "document_sha256": "eeb6bc6f7a3fce20654f977581e9718b124e70fc01ab358efebefe9afd9dc942", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": null, "scope": "Source declaration only; base weights, converters, runtimes and source integration material retain their own terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-grounded editorial synthesis with local Spark Qwen assistance and Codex review; no site model benchmark.", "answer": "Two saved runs found one exact-string needle at 63 percent depth in random filler, with 1034753 and 1035020 prompt tokens. Each completion is seven tokens, with zero cached prompt tokens. These are two different build/setup observations, not repeated measurements of one unchanged configuration or evidence of general long-document reasoning.", "sections": [ { "heading": "What does success mean here?", "paragraphs": [ "Two saved runs found one exact-string needle at 63 percent depth in random filler, with 1034753 and 1035020 prompt tokens. Each completion is seven tokens, with zero cached prompt tokens. These are two different build/setup observations, not repeated measurements of one unchanged configuration or evidence of general long-document reasoning." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json", "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json" ] }, { "heading": "How much latency and headroom were observed?", "paragraphs": [ "The first token took 830.4 seconds in the earlier run and 817.7 seconds in the final setup. During the final fill, minimum host MemAvailable was 1779 MiB on the head and 2542 MiB on the worker. The source describes engine swap policy, host reclaim settings, Engram page dropping and earlyoom thresholds. This is a tuned narrow-margin observation, not a safe default capacity." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md" ] }, { "heading": "What is still unknown?", "paragraphs": [ "Exact executed image digests and request sampling values are absent from the selected raw records. The author also reports slower prefill states and final-image runs below the best earlier result. One depth, one string and a seven-token output do not validate multiple needles, long generation, multimodal context, concurrent near-million-token requests or general answer correctness." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/KNOWN-LIMITS.md", "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md" ] }, { "heading": "What do the saved usage counts establish?", "paragraphs": [ "The two saved JSON records retain the actual prompt count, seven completion tokens, zero cached prompt tokens and zero reported reasoning tokens. These values establish the size and outcome of the performed requests. Zero reasoning tokens does not identify an unlogged request thinking flag, and a configured context limit is not an additional completed request." ], "citations": [ "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json" ] } ], "facts": [], "reported_results": [ { "label": "fd2365042f56 near-1M first token", "metric": "ttft", "value": 830.4, "unit": "s", "statistic": "single prompt, no variance estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "fd2365042f56", "input_tokens": 1034753, "output_tokens": 7, "cached_tokens": 0, "cache_state": "zero prefix-cache hit tokens; not a cold OS page-cache guarantee", "experiment_date": "2026-10-18T01:51:54Z", "needle_found": true, "reasoning_tokens": 0, "total_seconds": 835.8 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-needle-1m.json", "source_document_sha256": "9b81971c1378ab5b0bef466219c3a7aa7e2f78315bddbfc5bd0837cfec2fde40", "source_line": 8, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json#L8" }, { "label": "92aa710b533d near-1M first token", "metric": "ttft", "value": 817.7, "unit": "s", "statistic": "single prompt, no variance estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cached_tokens": 0, "cache_state": "zero prefix-cache hit tokens; not a cold OS page-cache guarantee", "experiment_date": "2026-10-18T04:47:07Z", "needle_found": true, "reasoning_tokens": 0, "total_seconds": 825.9 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "source_document_sha256": "c476006b019d46c68a334ad3763f3e8e0f0532d01dfc6fb800c043c225709ab1", "source_line": 8, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json#L8" }, { "label": "near-1M final head minimum MemAvailable", "metric": "minimum_memavailable", "value": 1779, "unit": "MiB", "statistic": "minimum observed in this one fill, not a safe capacity guarantee", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cache_state": "same final needle; cached_tokens 0", "experiment_date": "2026-10-18", "memory_kind": "minimum_host_MemAvailable_during_prefill", "node": "head", "earlyoom_limit_MiB_at_measurement": 1536, "earlyoom_later_limit_MiB": 1024, "OS_memory_controls_changed": true, "ordinary_host_baseline_inferred": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-INDEX.md", "source_document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "source_line": 59, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L59" }, { "label": "near-1M final worker minimum MemAvailable", "metric": "minimum_memavailable", "value": 2542, "unit": "MiB", "statistic": "minimum observed in this one fill, not a safe capacity guarantee", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cache_state": "same final needle; cached_tokens 0", "experiment_date": "2026-10-18", "memory_kind": "minimum_host_MemAvailable_during_prefill", "node": "worker", "earlyoom_limit_MiB_at_measurement": 1536, "earlyoom_later_limit_MiB": 1024, "OS_memory_controls_changed": true, "ordinary_host_baseline_inferred": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-INDEX.md", "source_document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "source_line": 59, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L59" } ], "source_document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e" }, "long_context_cases": [ { "id": "dsneedle-runs-2026-10-18-needle-1m.json", "classification": "source_reported_single_needle_success", "configured_limit_tokens": 1048576, "actual_input_tokens": 1034753, "actual_output_tokens": 7, "needle_depth_fraction": 0.63, "exact_match": true, "TTFT_seconds": 830.4, "cached_tokens": 0, "runtime_revision_prefix": "fd2365042f56", "executed_image_digest": null, "sampling": null, "quality_scope": "one needle at one depth; not broad reasoning, multi-needle, long-response or concurrency validation", "source_document_id": "dsneedle-runs-2026-10-18-needle-1m.json", "source_document_sha256": "9b81971c1378ab5b0bef466219c3a7aa7e2f78315bddbfc5bd0837cfec2fde40", "source_line": 3, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json#L3" }, { "id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "classification": "source_reported_single_needle_success", "configured_limit_tokens": 1048576, "actual_input_tokens": 1035020, "actual_output_tokens": 7, "needle_depth_fraction": 0.63, "exact_match": true, "TTFT_seconds": 817.7, "cached_tokens": 0, "runtime_revision_prefix": "92aa710b533d", "executed_image_digest": null, "sampling": null, "quality_scope": "one needle at one depth; not broad reasoning, multi-needle, long-response or concurrency validation", "source_document_id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "source_document_sha256": "c476006b019d46c68a334ad3763f3e8e0f0532d01dfc6fb800c043c225709ab1", "source_line": 3, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json#L3" } ], "memory_observations": [ { "record_id": "deepseek-1017-million-token-needle", "kind": "minimum_host_MemAvailable_during_final_fill", "head_MiB": 1779, "worker_MiB": 2542, "host_tuning_required_in_source": true, "safe_capacity_guarantee": false } ] } ] }, { "id": "deepseek-1017-quantization", "title": "Separate 1017 EXL3 runtime evidence from VQ artifact inspection", "needs": [ "catalog" ], "purpose": "Return complete identities and conditions; VQ serving speed and runtime RAM remain unmeasured, not borrowed from older V4.1.", "expression": "catalog.records.filter(r => ['deepseek-1017-dual-spark-quick','deepseek-1017-vq-artifact-inspection'].includes(r.id))", "expected_output": [ { "id": "deepseek-1017-dual-spark-quick", "title": "DeepSeek 1017: short-run EXL3 deployment and pool budget", "kind": "benchmark", "source_type": "community_operator_report", "summary": "The four selected C1 decode observations come from the raw quick-matrix CSVs, with 520-token thinking-off or 546-token effort-75 prompts and 64 actual output tokens. Each is one pass without variance estimates. Their first-to-last streamed-delta rate excludes initial prefill; end-to-end wall throughput remains a different field. This is not a long-context or sustained-response experiment.", "runtime": "TensorFold / vLLM", "runtime_topics": [ "TensorFold", "vLLM" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "DeepSeek-V4.1-Flash-1017", "model_family": "DeepSeek", "model_id": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_release_date": null, "model_artifact": { "id": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1" }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "kv_dtype": null, "context_limit_tokens": 262144, "experiment_date": null }, "test_profiles": [ { "label": "tensorfold C1 thinking off", "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv#L2", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "TensorFold Python 0.6.0 patched", "runtime_execution_revision": "c4646171139ee8a3c38103eaa1699dad226ec12b", "overlay_source_revision": "2dcce0303451da2a59f80e6c7413da8b3b0f7a57", "executed_image_digest": null, "thinking": false, "reasoning_effort": null, "input_tokens": 520, "actual_output_tokens": 64, "batch_wall_seconds": 2.4011132870218717, "end_to_end_output_wall_tps": 26.65430254620761, "TTFT_seconds": 1.313945789996069 } }, { "label": "tensorfold C1 thinking 75", "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv#L10", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "TensorFold Python 0.6.0 patched", "runtime_execution_revision": "c4646171139ee8a3c38103eaa1699dad226ec12b", "overlay_source_revision": "2dcce0303451da2a59f80e6c7413da8b3b0f7a57", "executed_image_digest": null, "thinking": true, "reasoning_effort": 75, "input_tokens": 546, "actual_output_tokens": 64, "batch_wall_seconds": 2.141774678020738, "end_to_end_output_wall_tps": 29.881761446140466, "TTFT_seconds": 1.326543439005036 } }, { "label": "vllm C1 thinking off", "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv#L2", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "vLLM + EXL3 overlay", "runtime_execution_revision": null, "overlay_source_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "executed_image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "thinking": false, "reasoning_effort": null, "input_tokens": 520, "actual_output_tokens": 64, "batch_wall_seconds": 2.5652991679962724, "end_to_end_output_wall_tps": 24.948357212460998, "TTFT_seconds": 0.8555283379973844 } }, { "label": "vllm C1 thinking 75", "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv#L10", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "vLLM + EXL3 overlay", "runtime_execution_revision": null, "overlay_source_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "executed_image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "thinking": true, "reasoning_effort": 75, "input_tokens": 546, "actual_output_tokens": 64, "batch_wall_seconds": 2.4199117650277913, "end_to_end_output_wall_tps": 26.447245277666145, "TTFT_seconds": 0.8985815939959139 } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "source_memory_and_endpoint_observations", "notes": [ "A source document, not an independent experiment count. Preserve measurement type and unknown fields." ], "limitations": [ "No independent Run AI Here model execution.", "Document revision is distinct from execution and weight identity.", "File bytes, host RSS, anonymous memory, MemAvailable, KV allocation and configured context are different quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "x00byte", "url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "platform": "github", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "README.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/README.md", "report_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/README.md", "document_sha256": "839509cc2365ea030d94360454437e66165c1087182ce6ccb85d23d85d5bb66d", "last_commit_at": "2026-10-17T18:27:21Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "dsdual-config-manifest.json", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "config/manifest.json", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/manifest.json", "document_sha256": "6b3da5443112253502befe2a866983a31d188af234aef5e03865a724dbe9b65a", "checked_on": "2026-10-18" }, { "document_id": "dsdual-config-tensorfold-knobs.env.example", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "config/tensorfold-knobs.env.example", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/tensorfold-knobs.env.example", "document_sha256": "e26dc4761458be238d048c5a03b524993e7c632c27f8457527e3ba0030bc58c0", "checked_on": "2026-10-18" }, { "document_id": "dsdual-config-vllm-knobs.env.example", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "config/vllm-knobs.env.example", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/vllm-knobs.env.example", "document_sha256": "ee4eb1535b45d28c47fdd5b136e4b04268a7955d2ac9ca83174afeb35c0d62d8", "checked_on": "2026-10-18" }, { "document_id": "dsdual-docs-deployment.md", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "docs/deployment.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/deployment.md", "document_sha256": "72fb210773604c15560e73d6db8ada144b0e953354201e42b3b7aebd4c018799", "checked_on": "2026-10-18" }, { "document_id": "dsdual-docs-benchmark-method.md", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "docs/benchmark-method.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/benchmark-method.md", "document_sha256": "33d83fe761ea2eaafaadee937fa212ffcec19d69a2528614776ad7fb1139ba3b", "checked_on": "2026-10-18" }, { "document_id": "dsdual-results-README.md", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "results/README.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/README.md", "document_sha256": "380393cc1e86d175f21af1c216b613a2aad741ed90c851b7e8f491bed5231acf", "checked_on": "2026-10-18" }, { "document_id": "dsdual-results-tensorfold-quick.csv", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "results/tensorfold-quick.csv", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv", "document_sha256": "3005b3be00581a962f9a269211eef2d598a6358d83c41e565ee1549b28e03c33", "checked_on": "2026-10-18" }, { "document_id": "dsdual-results-vllm-quick.csv", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "results/vllm-quick.csv", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv", "document_sha256": "befbf211d4f143798e12053ec21291cc77e0893284ccdb8d5a0aa19ef4722c6e", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": null, "scope": "Source declaration only; base weights, converters, runtimes and source integration material retain their own terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-grounded editorial synthesis with local Spark Qwen assistance and Codex review; no site model benchmark.", "answer": "The four selected C1 decode observations come from the raw quick-matrix CSVs, with 520-token thinking-off or 546-token effort-75 prompts and 64 actual output tokens. Each is one pass without variance estimates. Their first-to-last streamed-delta rate excludes initial prefill; end-to-end wall throughput remains a different field. This is not a long-context or sustained-response experiment.", "sections": [ { "heading": "What was measured?", "paragraphs": [ "The four selected C1 decode observations come from the raw quick-matrix CSVs, with 520-token thinking-off or 546-token effort-75 prompts and 64 actual output tokens. Each is one pass without variance estimates. Their first-to-last streamed-delta rate excludes initial prefill; end-to-end wall throughput remains a different field. This is not a long-context or sustained-response experiment." ], "citations": [ "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/benchmark-method.md", "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv", "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv" ] }, { "heading": "Which files and memory controls belong to this profile?", "paragraphs": [ "Both runtimes use the same pinned EXL3 conversion and separately pinned official 1017 Engrams. The TensorFold Python profile has eight request slots, a shared 524288-token pool, a 262144 per-request ceiling and configured 5/4 GiB memory floors. Floors are admission controls, not observed peak memory or a proof of full-window concurrency." ], "citations": [ "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/manifest.json", "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/deployment.md", "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/tensorfold-knobs.env.example" ] }, { "heading": "Can these rates be used for a VQ or million-token deployment?", "paragraphs": [ "No such transfer is established. The runtime, quantized tensors, request length and cache policy must stay attached to each observation. The text matrix predates the explicit native-image-routing flag; a later red-image smoke check confirms that path only. No comprehensive image or model-quality benchmark is claimed." ], "citations": [ "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/benchmark-method.md", "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/deployment.md" ] }, { "heading": "How does storage placement affect the deployment description?", "paragraphs": [ "The reported worker reads checkpoint files over a read-only NFS share, while active Engram lookups remain rank-local. The storage inventory includes separate source tables and packed files. Disk preparation, model residency and a shared KV budget must be recorded independently rather than adding all on-disk bytes into a claimed live memory requirement." ], "citations": [ "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/deployment.md" ] } ], "facts": [], "reported_results": [ { "label": "tensorfold C1 thinking off", "metric": "decode_rate", "value": 58.05409336065285, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "TensorFold Python 0.6.0 patched", "runtime_execution_revision": "c4646171139ee8a3c38103eaa1699dad226ec12b", "overlay_source_revision": "2dcce0303451da2a59f80e6c7413da8b3b0f7a57", "executed_image_digest": null, "thinking": false, "reasoning_effort": null, "input_tokens": 520, "actual_output_tokens": 64, "batch_wall_seconds": 2.4011132870218717, "end_to_end_output_wall_tps": 26.65430254620761, "TTFT_seconds": 1.313945789996069 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-tensorfold-quick.csv", "source_document_sha256": "3005b3be00581a962f9a269211eef2d598a6358d83c41e565ee1549b28e03c33", "source_line": 2, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv#L2" }, { "label": "tensorfold C1 thinking 75", "metric": "decode_rate", "value": 77.49652509362433, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "TensorFold Python 0.6.0 patched", "runtime_execution_revision": "c4646171139ee8a3c38103eaa1699dad226ec12b", "overlay_source_revision": "2dcce0303451da2a59f80e6c7413da8b3b0f7a57", "executed_image_digest": null, "thinking": true, "reasoning_effort": 75, "input_tokens": 546, "actual_output_tokens": 64, "batch_wall_seconds": 2.141774678020738, "end_to_end_output_wall_tps": 29.881761446140466, "TTFT_seconds": 1.326543439005036 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-tensorfold-quick.csv", "source_document_sha256": "3005b3be00581a962f9a269211eef2d598a6358d83c41e565ee1549b28e03c33", "source_line": 10, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv#L10" }, { "label": "vllm C1 thinking off", "metric": "decode_rate", "value": 36.8753802051545, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "vLLM + EXL3 overlay", "runtime_execution_revision": null, "overlay_source_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "executed_image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "thinking": false, "reasoning_effort": null, "input_tokens": 520, "actual_output_tokens": 64, "batch_wall_seconds": 2.5652991679962724, "end_to_end_output_wall_tps": 24.948357212460998, "TTFT_seconds": 0.8555283379973844 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-vllm-quick.csv", "source_document_sha256": "befbf211d4f143798e12053ec21291cc77e0893284ccdb8d5a0aa19ef4722c6e", "source_line": 2, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv#L2" }, { "label": "vllm C1 thinking 75", "metric": "decode_rate", "value": 41.447290817386964, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "vLLM + EXL3 overlay", "runtime_execution_revision": null, "overlay_source_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "executed_image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "thinking": true, "reasoning_effort": 75, "input_tokens": 546, "actual_output_tokens": 64, "batch_wall_seconds": 2.4199117650277913, "end_to_end_output_wall_tps": 26.447245277666145, "TTFT_seconds": 0.8985815939959139 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-vllm-quick.csv", "source_document_sha256": "befbf211d4f143798e12053ec21291cc77e0893284ccdb8d5a0aa19ef4722c6e", "source_line": 10, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv#L10" } ], "source_document_sha256": "839509cc2365ea030d94360454437e66165c1087182ce6ccb85d23d85d5bb66d" } }, { "id": "deepseek-1017-vq-artifact-inspection", "title": "DeepSeek 1017: VQ file identity before runtime claims", "kind": "deployment", "source_type": "community_operator_report", "summary": "The converter reports a 1000-tensor base, its exact assembled file size and a successful loader binding check. The separate vision sidecar and official 1017 Engram files retain individual identities. These are artifact and loader observations. The selected source explicitly has no 1017-specific serving speed, peak RAM or model-quality measurement.", "runtime": "ds4-dfm-rs", "runtime_topics": [ "ds4-dfm-rs" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "DeepSeek-V4.1-Flash-1017", "model_family": "DeepSeek", "model_id": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "model_release_date": null, "model_artifact": { "id": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5" }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": null, "node_count": null, "parallelism": null, "kv_dtype": null, "context_limit_tokens": 1048576, "experiment_date": "2026-10-17 status in card" }, "test_profiles": [ { "label": "1017 VQ base GGUF", "evidence_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md#L97", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "model_revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "base_model_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "quantizer_revision": "7fb26cc332d740416792204a29c7d13d2986caf5", "vision_converter_revision": "e4ea07889144d85acd33b2490ec4f44032644fb0", "quantization": "YoungAi vq8sh14-q4k-mtpnative; routed-expert VQ, q4_K skeleton; native MTP towers", "measurement_kind": "publisher artifact layout inspection, not runtime evaluation", "experiment_date": "2026-10-17 status in card", "system": null, "engine": "target ds4-dfm-rs; loader check passed, serving unmeasured on 1017", "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "context_limit_tokens": 1048576, "context_limit_origin": "model metadata, not measured serving result", "speed_1017_measured": false, "quality_1017_measured": false, "peak_runtime_memory_measured": false, "base_model_speed_inherited": false, "filename": "DeepSeek-V4.1-Flash-vq8sh14-q4k-mtpnative.gguf", "memory_kind": "artifact_disk_file_bytes", "Engram_included": false } }, { "label": "1017 BF16 vision sidecar", "evidence_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md#L98", "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "model_revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "base_model_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "quantizer_revision": "7fb26cc332d740416792204a29c7d13d2986caf5", "vision_converter_revision": "e4ea07889144d85acd33b2490ec4f44032644fb0", "quantization": "YoungAi vq8sh14-q4k-mtpnative; routed-expert VQ, q4_K skeleton; native MTP towers", "measurement_kind": "publisher artifact layout inspection, not runtime evaluation", "experiment_date": "2026-10-17 status in card", "system": null, "engine": "target ds4-dfm-rs; loader check passed, serving unmeasured on 1017", "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "context_limit_tokens": 1048576, "context_limit_origin": "model metadata, not measured serving result", "speed_1017_measured": false, "quality_1017_measured": false, "peak_runtime_memory_measured": false, "base_model_speed_inherited": false, "filename": "DeepSeek-V4.1-Flash.vision.gguf", "memory_kind": "artifact_disk_file_bytes", "Engram_included": false } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "artifact_layout_only", "notes": [ "A source document, not an independent experiment count. Preserve measurement type and unknown fields." ], "limitations": [ "No independent Run AI Here model execution.", "Document revision is distinct from execution and weight identity.", "File bytes, host RSS, anonymous memory, MemAvailable, KV allocation and configured context are different quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "Baekpica", "url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "platform": "huggingface", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "README.md", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md", "report_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md", "document_sha256": "b9f85bae000a8aa0e270555d17b1f9bd859a79bc3f5f2d7fde7eb00607cc990f", "last_commit_at": "2026-10-17T11:05:38.000Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "ds1017vq-RECIPE.md", "parent_record_id": "deepseek-1017-vq-artifact-inspection", "publisher": "Baekpica", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "RECIPE.md", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/RECIPE.md", "document_sha256": "1b66bfc44386bd6b708b1d4250cdd0dd825d2b45eee18c74d2bebfbff9e62486", "checked_on": "2026-10-18" }, { "document_id": "ds1017vq-SOURCE.md", "parent_record_id": "deepseek-1017-vq-artifact-inspection", "publisher": "Baekpica", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "SOURCE.md", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SOURCE.md", "document_sha256": "d039fb41a620380e352de9794271a6b2855e5d6931979d43695f4d3a236d443e", "checked_on": "2026-10-18" }, { "document_id": "ds1017vq-SHA256SUMS", "parent_record_id": "deepseek-1017-vq-artifact-inspection", "publisher": "Baekpica", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "SHA256SUMS", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SHA256SUMS", "document_sha256": "6b98a0a8923558f69ad53a9d60c29332b984a59f8109a6115c7c88e684763b19", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "mit", "scope": "Source declaration only; base weights, converters, runtimes and source integration material retain their own terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-grounded editorial synthesis with local Spark Qwen assistance and Codex review; no site model benchmark.", "answer": "The converter reports a 1000-tensor base, its exact assembled file size and a successful loader binding check. The separate vision sidecar and official 1017 Engram files retain individual identities. These are artifact and loader observations. The selected source explicitly has no 1017-specific serving speed, peak RAM or model-quality measurement.", "sections": [ { "heading": "Which facts are confirmed by the publisher?", "paragraphs": [ "The converter reports a 1000-tensor base, its exact assembled file size and a successful loader binding check. The separate vision sidecar and official 1017 Engram files retain individual identities. These are artifact and loader observations. The selected source explicitly has no 1017-specific serving speed, peak RAM or model-quality measurement." ], "citations": [ "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md", "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SHA256SUMS" ] }, { "heading": "Why are old names unsafe identifiers?", "paragraphs": [ "The 1017 Engram shard names and byte sizes match the older release, while their hashes differ. The vision sidecar also keeps its name and size with different contents. Use revision and SHA-256 alongside paths. The quantizer and vision converter have separate pinned commits, and domain sidecars trained against the older base are not treated as interchangeable." ], "citations": [ "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SOURCE.md", "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/RECIPE.md" ] }, { "heading": "Does a small file prove a small runtime footprint?", "paragraphs": [ "The base excludes its Engram files and does not describe all live buffers, caches, page residency or service overhead. Its native context field comes from model metadata. Old V4.1 speed and fidelity numbers in the card remain old-checkpoint reference results; they are not published as this 1017 artifact’s measured performance." ], "citations": [ "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md", "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/RECIPE.md" ] }, { "heading": "Does the GGUF suffix certify generic loader compatibility?", "paragraphs": [ "This artifact uses custom VQ payload storage and targets the named ds4-dfm-rs or YoungAi path. The file extension alone does not show that an arbitrary llama.cpp or Transformers installation can load it. Keep the converter commits, recipe string and stated target runtime rather than generalizing compatibility from the container name." ], "citations": [ "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/RECIPE.md" ] } ], "facts": [], "reported_results": [ { "label": "1017 VQ base GGUF", "metric": "artifact_size", "value": 113556639424, "unit": "bytes", "statistic": "source-reported file bytes; not a RAM requirement", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "model_revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "base_model_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "quantizer_revision": "7fb26cc332d740416792204a29c7d13d2986caf5", "vision_converter_revision": "e4ea07889144d85acd33b2490ec4f44032644fb0", "quantization": "YoungAi vq8sh14-q4k-mtpnative; routed-expert VQ, q4_K skeleton; native MTP towers", "measurement_kind": "publisher artifact layout inspection, not runtime evaluation", "experiment_date": "2026-10-17 status in card", "system": null, "engine": "target ds4-dfm-rs; loader check passed, serving unmeasured on 1017", "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "context_limit_tokens": 1048576, "context_limit_origin": "model metadata, not measured serving result", "speed_1017_measured": false, "quality_1017_measured": false, "peak_runtime_memory_measured": false, "base_model_speed_inherited": false, "filename": "DeepSeek-V4.1-Flash-vq8sh14-q4k-mtpnative.gguf", "memory_kind": "artifact_disk_file_bytes", "Engram_included": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "ds1017vq-readme", "source_document_sha256": "b9f85bae000a8aa0e270555d17b1f9bd859a79bc3f5f2d7fde7eb00607cc990f", "source_line": 97, "evidence_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md#L97" }, { "label": "1017 BF16 vision sidecar", "metric": "artifact_size", "value": 971066016, "unit": "bytes", "statistic": "source-reported file bytes; not a RAM requirement", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "model_revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "base_model_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "quantizer_revision": "7fb26cc332d740416792204a29c7d13d2986caf5", "vision_converter_revision": "e4ea07889144d85acd33b2490ec4f44032644fb0", "quantization": "YoungAi vq8sh14-q4k-mtpnative; routed-expert VQ, q4_K skeleton; native MTP towers", "measurement_kind": "publisher artifact layout inspection, not runtime evaluation", "experiment_date": "2026-10-17 status in card", "system": null, "engine": "target ds4-dfm-rs; loader check passed, serving unmeasured on 1017", "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "context_limit_tokens": 1048576, "context_limit_origin": "model metadata, not measured serving result", "speed_1017_measured": false, "quality_1017_measured": false, "peak_runtime_memory_measured": false, "base_model_speed_inherited": false, "filename": "DeepSeek-V4.1-Flash.vision.gguf", "memory_kind": "artifact_disk_file_bytes", "Engram_included": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "ds1017vq-readme", "source_document_sha256": "b9f85bae000a8aa0e270555d17b1f9bd859a79bc3f5f2d7fde7eb00607cc990f", "source_line": 98, "evidence_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md#L98" } ], "source_document_sha256": "b9f85bae000a8aa0e270555d17b1f9bd859a79bc3f5f2d7fde7eb00607cc990f" }, "memory_observations": [ { "record_id": "deepseek-1017-vq-artifact-inspection", "kind": "file_inventory_not_RAM", "base_bytes": 113556639424, "vision_bytes": 971066016, "Engram_shard_bytes": [ 101535150936, 101537926640 ], "actual_peak_RAM": null, "speed_and_quality_1017": null } ] } ] }, { "id": "qwen-flash-next-spark-concurrency", "title": "Read both Spark concurrency and cold-prefix reports", "needs": [ "catalog" ], "purpose": "Keep distinct hardware, checkpoint releases, exact request mix, trial units, SD and configured capacity together.", "expression": "catalog.records.filter(r => ['qwen38-flash-next-sglang-concurrency','qwen38-flash-next-tp2-concurrency-context'].includes(r.id))", "expected_output": [ { "id": "qwen38-flash-next-sglang-concurrency", "title": "Qwen Flash-Next: SGLang concurrency and latency", "kind": "benchmark", "source_type": "community_operator_report", "summary": "One 128 GB unified-memory DGX Spark serves senfu NVFP4 at its full recorded revision through SGLang commit 4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b. The recipe keeps PLE on NVMe through mmap, uses an FP8 dense path and an NVFP4 LM head, and disables MTP. The repository documentation pin is separate from the engine and weight revisions.", "runtime": "SGLang", "runtime_topics": [ "SGLang" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "Qwen3.8-Flash-Next", "model_family": "Qwen", "model_id": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_release_date": null, "model_artifact": { "id": "senfu/Qwen3.8-Flash-Next-NVFP4", "revision": "5d37b3b3711d8406174b96ff950c0aa16324b266" }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "parallelism": null, "kv_dtype": null, "context_limit_tokens": null, "experiment_date": null }, "test_profiles": [ { "label": "single Spark senfu C1 aggregate_output_rate", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L9", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 1, "repetitions": 3, "summary_kind": "median of three trials" } }, { "label": "single Spark senfu C1 median_ttft", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L9", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 1, "repetitions": 3, "summary_kind": "median of three trials" } }, { "label": "single Spark senfu C1 median_time_per_output_token", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L9", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 1, "repetitions": 3, "summary_kind": "median of three trials" } }, { "label": "single Spark senfu C8 aggregate_output_rate", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L10", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 8, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C8 median_ttft", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L10", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 8, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C8 median_time_per_output_token", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L10", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 8, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C16 aggregate_output_rate", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L11", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 16, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C16 median_ttft", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L11", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 16, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C16 median_time_per_output_token", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L11", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 16, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C32 aggregate_output_rate", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L12", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 32, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C32 median_ttft", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L12", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 32, "repetitions": 1, "summary_kind": "single trial" } }, { "label": "single Spark senfu C32 median_time_per_output_token", "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L12", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 32, "repetitions": 1, "summary_kind": "single trial" } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "source_request_latency_memory_and_gate_observations", "notes": [ "One source document can contain different experiments and versions; not an independent experiment count." ], "limitations": [ "No independent site model execution.", "Missing sampling, exact executed image digests and measurement boundaries remain null.", "Configured window, completed input, scheduler ceiling and measured full-window concurrency are separate quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "nabe2030", "url": "https://github.com/nabe2030/flashnext-dgx-spark", "repository": "nabe2030/flashnext-dgx-spark", "platform": "github", "revision": "8a161f2bee2e2d85b689a2d457ae68bead3929a4", "document_path": "README.md", "pinned_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/README.md", "report_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/README.md", "document_sha256": "8a27445102511702ff42fa60cfcc469583de37c4c0bd031c43cd40db7c0db9f4", "last_commit_at": "2026-09-21T10:13:50Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "qwen-sglang-results", "parent_record_id": "qwen38-flash-next-sglang-concurrency", "publisher": "nabe2030", "repository": "nabe2030/flashnext-dgx-spark", "revision": "8a161f2bee2e2d85b689a2d457ae68bead3929a4", "document_path": "results/speed_senfu_vs_radixark.md", "pinned_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md", "document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "apache-2.0", "scope": "Source declaration; models, runtime and dependencies retain their exact terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-based interpretation; local Spark Qwen draft with Codex factual review, no site model benchmark.", "answer": "One 128 GB unified-memory DGX Spark serves senfu NVFP4 at its full recorded revision through SGLang commit 4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b. The recipe keeps PLE on NVMe through mmap, uses an FP8 dense path and an NVFP4 LM head, and disables MTP. The repository documentation pin is separate from the engine and weight revisions.", "sections": [ { "heading": "Which checkpoint and runtime were tested?", "paragraphs": [ "One 128 GB unified-memory DGX Spark serves senfu NVFP4 at its full recorded revision through SGLang commit 4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b. The recipe keeps PLE on NVMe through mmap, uses an FP8 dense path and an NVFP4 LM head, and disables MTP. The repository documentation pin is separate from the engine and weight revisions." ], "citations": [ "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/README.md" ] }, { "heading": "What workload defines the concurrency curve?", "paragraphs": [ "The selected ShareGPT-200 sweep uses vllm bench serve, thinking off and explicitly disabled radix caching, after one discarded warmup request. C1 is the median of three trials; C8, C16 and C32 are single trials. Exact prompt/output distributions, sampling and per-sweep window are not reported. All selected senfu trials completed 200 requests without errors." ], "citations": [ "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md" ] }, { "heading": "How should throughput and latency be read together?", "paragraphs": [ "Aggregate output rises with concurrency, while the reported median TTFT and time per output token also rise. Read all three metrics with their own timing definitions. Dividing aggregate rate by concurrency does not reconstruct a measured individual stream. Single high-load trials provide no variability estimate, and results from another checkpoint or engine cannot isolate a pure scheduler effect." ], "citations": [ "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md" ] }, { "heading": "Does the recipe prove simultaneous long windows?", "paragraphs": [ "The 49152-token window, 393216 total tokens and eight slots describe one published long-context configuration, not a saved all-slots-full workload. The separate 3.04 GB KV allocation at 128K keeps the original GB label; it is not whole-model peak RAM. The source reports 128K stability but supplies no complete long-request latency or memory series for every concurrency level." ], "citations": [ "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/README.md" ] } ], "facts": [], "reported_results": [ { "label": "single Spark senfu C1 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 26.97, "unit": "tokens/s", "statistic": "median of three trials", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 1, "repetitions": 3, "summary_kind": "median of three trials" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 9, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L9" }, { "label": "single Spark senfu C1 median_ttft", "metric": "median_ttft", "value": 221.23, "unit": "ms", "statistic": "median of three trials", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 1, "repetitions": 3, "summary_kind": "median of three trials" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 9, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L9" }, { "label": "single Spark senfu C1 median_time_per_output_token", "metric": "median_time_per_output_token", "value": 36.14, "unit": "ms", "statistic": "median of three trials", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 1, "repetitions": 3, "summary_kind": "median of three trials" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 9, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L9" }, { "label": "single Spark senfu C8 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 67.68, "unit": "tokens/s", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 8, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 10, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L10" }, { "label": "single Spark senfu C8 median_ttft", "metric": "median_ttft", "value": 526.78, "unit": "ms", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 8, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 10, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L10" }, { "label": "single Spark senfu C8 median_time_per_output_token", "metric": "median_time_per_output_token", "value": 113.46, "unit": "ms", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 8, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 10, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L10" }, { "label": "single Spark senfu C16 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 77.48, "unit": "tokens/s", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 16, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 11, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L11" }, { "label": "single Spark senfu C16 median_ttft", "metric": "median_ttft", "value": 686.27, "unit": "ms", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 16, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 11, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L11" }, { "label": "single Spark senfu C16 median_time_per_output_token", "metric": "median_time_per_output_token", "value": 198, "unit": "ms", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 16, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 11, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L11" }, { "label": "single Spark senfu C32 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 89.35, "unit": "tokens/s", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 32, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 12, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L12" }, { "label": "single Spark senfu C32 median_ttft", "metric": "median_ttft", "value": 1017.98, "unit": "ms", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 32, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 12, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L12" }, { "label": "single Spark senfu C32 median_time_per_output_token", "metric": "median_time_per_output_token", "value": 343.57, "unit": "ms", "statistic": "single trial", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "senfu/Qwen3.8-Flash-Next-NVFP4", "model_revision": "5d37b3b3711d8406174b96ff950c0aa16324b266", "engine": "SGLang", "runtime_execution_revision": "4fb9b5b5bad41ac7de5586ec812c421fe3aa2b8b", "executed_image_digest": null, "system": "one DGX Spark GB10, 128 GB unified memory; NVMe file-backed PLE", "node_count": 1, "weight_precision": "NVFP4 experts and LM head; FP8 dense path", "kv_dtype": null, "draft_tokens": 0, "thinking": false, "temperature": null, "top_p": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "request_count": 200, "cache_state": "radix/prefix cache explicitly disabled; one warmup discarded", "harness": "vllm bench serve, ShareGPT 200 prompts", "experiment_date": null, "context_limit_tokens": null, "executed_context_for_each_sweep_unreported": true, "PLE_backing": "NVMe mmap", "exact_prompt_and_output_distributions_unreported": true, "concurrency": 32, "repetitions": 1, "summary_kind": "single trial" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-sglang-results", "source_document_sha256": "cf77f1feb1a37ab7fcdd8073ca0ff3b5fe78b8c47cf261fd917ae69fcb018297", "source_line": 12, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/results/speed_senfu_vs_radixark.md#L12" } ], "source_document_sha256": "8a27445102511702ff42fa60cfcc469583de37c4c0bd031c43cd40db7c0db9f4" }, "memory_observations": [ { "kind": "source_reported_KV_allocation_at_context", "context_limit_tokens": 131072, "KV_source_GB": 3.04, "GB_byte_convention_verified": false, "whole_model_peak_RAM": null, "source_document_id": "qwen-sglang-readme", "source_document_sha256": "8a27445102511702ff42fa60cfcc469583de37c4c0bd031c43cd40db7c0db9f4", "source_line": 19, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/README.md#L19" }, { "kind": "source_recipe_slots_not_a_completed_long_batch", "per_request_window_tokens": 49152, "shared_total_tokens": 393216, "request_slots": 8, "all_slots_full_simultaneously_proven": false, "source_document_id": "qwen-sglang-readme", "source_document_sha256": "8a27445102511702ff42fa60cfcc469583de37c4c0bd031c43cd40db7c0db9f4", "source_line": 134, "evidence_url": "https://github.com/nabe2030/flashnext-dgx-spark/blob/8a161f2bee2e2d85b689a2d457ae68bead3929a4/README.md#L134" } ] }, { "id": "qwen38-flash-next-tp2-concurrency-context", "title": "Qwen Flash-Next: TP2 concurrency and cold-prefix cohorts", "kind": "benchmark", "source_type": "community_operator_report", "summary": "The k76 v2.0.0 concurrency series uses the earlier 16c9bd54788d12390838a65ce4a4ecda97fa5f1d GDN-MSE checkpoint, while v2.2.0 cold-prefix measurements use 03f4a0570496bbe741189f4a2a7d611679e85406. Both employ two directly connected Sparks, vLLM plus b12x and mixed NVFP4/MXFP8 paths. Release image digests and abbreviated engine commits do not establish an exact executed image digest.", "runtime": "vLLM + b12x", "runtime_topics": [ "vLLM" ], "hardware_topics": [ "spark", "nvidia" ], "platform_topics": [ "linux" ], "model_label": "Qwen3.8-Flash-Next", "model_family": "Qwen", "model_id": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "model_release_date": null, "model_artifact": { "id": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "revision": null }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "kv_dtype": "fp8", "context_limit_tokens": 262144, "experiment_date": "2026-10-10" }, "test_profiles": [ { "label": "TP2 v2.0.0 C1 2048 per_request_decode_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1142", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 1, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.0.0 C4 2048 per_request_decode_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1146", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 4, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.0.0 C8 2048 per_request_decode_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1148", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 8, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.0.0 C16 2048 per_request_decode_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1150", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 16, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.0.0 C1 2048 aggregate_output_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1152", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 1, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.0.0 C4 2048 aggregate_output_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1156", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 4, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.0.0 C8 2048 aggregate_output_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1158", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 8, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.0.0 C16 2048 aggregate_output_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1160", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 16, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.2.0 C1 16384 ttft", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1262", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "ttft_s", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 16384, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.2.0 C1 16384 prefill_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1263", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "pp", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 16384, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.2.0 C1 65536 ttft", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1264", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "ttft_s", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 65536, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.2.0 C1 65536 prefill_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1265", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "pp", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 65536, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.2.0 C1 131072 ttft", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1266", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "ttft_s", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 131072, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false } }, { "label": "TP2 v2.2.0 C1 131072 prefill_rate", "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1267", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "pp", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 131072, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "source_request_latency_memory_and_gate_observations", "notes": [ "One source document can contain different experiments and versions; not an independent experiment count." ], "limitations": [ "No independent site model execution.", "Missing sampling, exact executed image digests and measurement boundaries remain null.", "Configured window, completed input, scheduler ceiling and measured full-window concurrency are separate quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "ursuciprian", "url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2", "repository": "ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2", "platform": "github", "revision": "b384f4a0e3ff0d563980befe52eb3e6964a5b6b1", "document_path": "README.md", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/README.md", "report_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/README.md", "document_sha256": "f3bd45cb1f01c287ef00f97b98e0d5c5d66576d5be34baaefce5c108242ad1cf", "last_commit_at": "2026-10-17T10:48:19Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "qwen-tp2-csv", "parent_record_id": "qwen38-flash-next-tp2-concurrency-context", "publisher": "ursuciprian", "repository": "ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2", "revision": "b384f4a0e3ff0d563980befe52eb3e6964a5b6b1", "document_path": "docs/data/capability.csv", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv", "document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "checked_on": "2026-10-18" }, { "document_id": "qwen-tp2-method", "parent_record_id": "qwen38-flash-next-tp2-concurrency-context", "publisher": "ursuciprian", "repository": "ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2", "revision": "b384f4a0e3ff0d563980befe52eb3e6964a5b6b1", "document_path": "docs/BENCHMARKS.md", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/BENCHMARKS.md", "document_sha256": "5645ff4e129b3e2766672e092c65333b073b5550ad89aa6374cbd81ce1da7776", "checked_on": "2026-10-18" }, { "document_id": "qwen-tp2-versions", "parent_record_id": "qwen38-flash-next-tp2-concurrency-context", "publisher": "ursuciprian", "repository": "ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2", "revision": "b384f4a0e3ff0d563980befe52eb3e6964a5b6b1", "document_path": "VERSIONS.md", "pinned_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/VERSIONS.md", "document_sha256": "eb9ec6435802430a1be7302b445c67ceb48da9cf5e1ba32e939b5f7aaf9b100e", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "apache-2.0", "scope": "Source declaration; models, runtime and dependencies retain their exact terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-based interpretation; local Spark Qwen draft with Codex factual review, no site model benchmark.", "answer": "The k76 v2.0.0 concurrency series uses the earlier 16c9bd54788d12390838a65ce4a4ecda97fa5f1d GDN-MSE checkpoint, while v2.2.0 cold-prefix measurements use 03f4a0570496bbe741189f4a2a7d611679e85406. Both employ two directly connected Sparks, vLLM plus b12x and mixed NVFP4/MXFP8 paths. Release image digests and abbreviated engine commits do not establish an exact executed image digest.", "sections": [ { "heading": "Why keep releases separate?", "paragraphs": [ "The k76 v2.0.0 concurrency series uses the earlier 16c9bd54788d12390838a65ce4a4ecda97fa5f1d GDN-MSE checkpoint, while v2.2.0 cold-prefix measurements use 03f4a0570496bbe741189f4a2a7d611679e85406. Both employ two directly connected Sparks, vLLM plus b12x and mixed NVFP4/MXFP8 paths. Release image digests and abbreviated engine commits do not establish an exact executed image digest." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/VERSIONS.md" ] }, { "heading": "What does the k76 concurrency series measure?", "paragraphs": [ "The selected cells use llama-benchy 0.4.1.dev4+g0d4de4271, 2048 input, 512 requested output tokens, temperature 1.0, top-p 0.95, top-k 20 and thinking on. Prefix caching is enabled. Means and SD cover three runs in one boot, not independent boots. Aggregate output includes prompt-processing wall time; individual decode begins after the first token." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv", "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/BENCHMARKS.md" ] }, { "heading": "What does the later cold-prefix arm measure?", "paragraphs": [ "At C1 and temperature zero, v2.2.0 requests one output token after 16384, 65536 or 131072 input tokens. Four boots each contain three cold prompts. The SD describes variation between boot means. These observations measure first-token latency and prefill, not sustained generation, broad long-document accuracy or multi-user full-window serving." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv", "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/BENCHMARKS.md" ] }, { "heading": "Which conclusions are outside these data?", "paragraphs": [ "The two arms cannot be joined into a single unchanged deployment curve. Current request slots and KV capacity are configuration limits, and cold prefix does not mean cold filesystem pages or cold compilation state. Complete executed digests and some runtime identities remain null. The publisher is already represented by a separate one-Spark document in the catalog; this new document is not a new independent researcher." ], "citations": [ "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/README.md", "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/VERSIONS.md" ] } ], "facts": [], "reported_results": [ { "label": "TP2 v2.0.0 C1 2048 per_request_decode_rate", "metric": "per_request_decode_rate", "value": 88.697, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 2.562, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 1, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1142, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1142" }, { "label": "TP2 v2.0.0 C4 2048 per_request_decode_rate", "metric": "per_request_decode_rate", "value": 48.631, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 1.533, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 4, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1146, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1146" }, { "label": "TP2 v2.0.0 C8 2048 per_request_decode_rate", "metric": "per_request_decode_rate", "value": 33.236, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 0.977, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 8, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1148, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1148" }, { "label": "TP2 v2.0.0 C16 2048 per_request_decode_rate", "metric": "per_request_decode_rate", "value": 22.142, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 0.362, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_req", "source_campaign": "k76-capability-matrix", "concurrency": 16, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1150, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1150" }, { "label": "TP2 v2.0.0 C1 2048 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 88.697, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 2.562, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 1, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1152, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1152" }, { "label": "TP2 v2.0.0 C4 2048 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 156.645, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 4.558, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 4, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1156, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1156" }, { "label": "TP2 v2.0.0 C8 2048 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 172.213, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 16.348, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 8, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1158, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1158" }, { "label": "TP2 v2.0.0 C16 2048 aggregate_output_rate", "metric": "aggregate_output_rate", "value": 231.294, "unit": "tokens/s", "statistic": "mean ± sd of 3 runs, one boot", "uncertainty": { "standard_deviation": 4.91, "kind": "SD across runs in one boot" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "16c9bd54788d12390838a65ce4a4ecda97fa5f1d", "recipe_release": "v2.0.0", "experiment_date": "2026-10-10", "harness": "llama-benchy 0.4.1.dev4+g0d4de4271", "source_run": "2x:results/k76-capability-matrix-20261010-1059/2x/d0.json", "source_metric": "tg_total", "source_campaign": "k76-capability-matrix", "concurrency": 16, "input_tokens": 2048, "output_tokens": 512, "temperature": 1, "top_p": 0.95, "top_k": 20, "thinking": true, "cache_state": "prefix caching enabled; fresh 2048-token task, not a cold-cache campaign", "repetitions": "mean ± sd of 3 runs, one boot", "aggregate_denominator": "all output tokens / wall time including prompt processing", "per_request_denominator": "one output stream after first token", "release_image_digest": "sha256:8abb60eeeb79ca13680341c17d87975801da1218c06df2505856b23da7ed1821", "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1160, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1160" }, { "label": "TP2 v2.2.0 C1 16384 ttft", "metric": "ttft", "value": 5.487, "unit": "s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "standard_deviation": 0.045, "kind": "SD between boot means; four boots x three cold prompts" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "ttft_s", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 16384, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1262, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1262" }, { "label": "TP2 v2.2.0 C1 16384 prefill_rate", "metric": "prefill_rate", "value": 2986, "unit": "tokens/s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "standard_deviation": 24.6, "kind": "SD between boot means; four boots x three cold prompts" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "pp", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 16384, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1263, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1263" }, { "label": "TP2 v2.2.0 C1 65536 ttft", "metric": "ttft", "value": 24.041, "unit": "s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "standard_deviation": 0.057, "kind": "SD between boot means; four boots x three cold prompts" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "ttft_s", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 65536, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1264, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1264" }, { "label": "TP2 v2.2.0 C1 65536 prefill_rate", "metric": "prefill_rate", "value": 2726.1, "unit": "tokens/s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "standard_deviation": 6.4, "kind": "SD between boot means; four boots x three cold prompts" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "pp", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 65536, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1265, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1265" }, { "label": "TP2 v2.2.0 C1 131072 ttft", "metric": "ttft", "value": 54.037, "unit": "s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "standard_deviation": 0.256, "kind": "SD between boot means; four boots x three cold prompts" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "ttft_s", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 131072, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1266, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1266" }, { "label": "TP2 v2.2.0 C1 131072 prefill_rate", "metric": "prefill_rate", "value": 2425.6, "unit": "tokens/s", "statistic": "mean of 4 boots x 3 cold prompts, sd between boots", "uncertainty": { "standard_deviation": 11.5, "kind": "SD between boot means; four boots x three cold prompts" }, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "ursuciprian/Qwen3.8-Flash-Next-NVFP4-GDN-MSE", "engine": "vLLM + b12x", "engine_version": null, "runtime_execution_revision": null, "executed_image_digest": null, "system": "two DGX Spark GB10, 128 GB unified memory each; ConnectX-7 direct-cabled", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVFP4 routed experts and decode GDN; MXFP8 dense/attention and prefill GDN copies", "extra_prefill_revision": "7c4f1bc1a2d6847e0cbc01ac6b823f00251de8dd", "kv_dtype": "fp8", "draft_tokens": 4, "context_limit_tokens": 262144, "configured_max_requests": 16, "actual_output_tokens_per_request": null, "GPU_clock_cap_MHz": null, "model_revision": "03f4a0570496bbe741189f4a2a7d611679e85406", "recipe_release": "v2.2.0", "experiment_date": "2026-10-15", "harness": "cold.py (k101)", "source_run": "1x:results/k101s-1x-gdn-prefill-views-ab-20261015-1600/tp2/2x/", "source_metric": "pp", "source_campaign": "promotion A/B", "concurrency": 1, "input_tokens": 131072, "output_tokens": 1, "temperature": 0, "top_p": null, "top_k": null, "thinking": null, "cache_state": "cold prefix with no hit, not a cold OS page-cache claim", "repetitions": "mean of 4 boots x 3 cold prompts, sd between boots", "aggregate_denominator": null, "per_request_denominator": null, "release_image_digest": null, "release_digest_is_proven_executed_digest": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-tp2-csv", "source_document_sha256": "a82bffe4f0e33f4243db9549c02fb8e9379e5faa62825a35a7b087cd30b4996e", "source_line": 1267, "evidence_url": "https://github.com/ursuciprian/qwen3.8-flash-next-dgx-spark-tp-2/blob/b384f4a0e3ff0d563980befe52eb3e6964a5b6b1/docs/data/capability.csv#L1267" } ], "source_document_sha256": "f3bd45cb1f01c287ef00f97b98e0d5c5d66576d5be34baaefce5c108242ad1cf" } } ] }, { "id": "qwen-flash-next-5090-gates", "title": "Read actual 5090 context gates and unpaired memory telemetry", "needs": [ "catalog" ], "purpose": "Retain the 104805-token retrieval, 32K-only peaks, unknown sampling scope and separate ngram speculation/embedding states.", "expression": "catalog.records.filter(r => r.id === 'qwen38-flash-next-5090-offload-gates')", "expected_output": [ { "id": "qwen38-flash-next-5090-offload-gates", "title": "Qwen Flash-Next: RTX 5090 CPU offload and long-context gates", "kind": "benchmark", "source_type": "community_operator_report", "summary": "The pinned MXFP4 GGUF runs through ik_llama.cpp merge d854dfd36a7262fc46cb7542716b04a9f6d4b47d on one RTX 5090 with WSL2, a 128 GiB-class host and 96 GiB VM ceiling. The baseline puts 38 expert layers on CPU, keeps mmap, disables the CUDA pinned-host override and uses Q8_0 KV. One request and one slot are used throughout; this is not full-model GPU residence.", "runtime": "ik_llama.cpp", "runtime_topics": [ "llama.cpp" ], "hardware_topics": [ "nvidia" ], "platform_topics": [ "linux", "windows" ], "model_label": "Qwen3.8-Flash-Next", "model_family": "Qwen", "model_id": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_release_date": null, "model_artifact": { "id": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "revision": "4a768bf4458a83e1c80667b4d818ec721080546e" }, "evidence_state": "source_reported", "site_reproduced": false, "record_type": "curated_reference", "performance_ranking_allowed": false, "metrics": null, "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results", "conditions": { "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "reported_setup": { "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "parallelism": null, "kv_dtype": "Q8_0", "context_limit_tokens": 32768, "experiment_date": null }, "test_profiles": [ { "label": "32K needle gate prefill_rate", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L53", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "workload": "32K needle gate", "memory_cohort": "completed 32K monitoring" } }, { "label": "32K needle gate decode_rate", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L53", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "workload": "32K needle gate", "memory_cohort": "completed 32K monitoring" } }, { "label": "128K server smoke prefill_rate", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L54", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 61, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K server smoke", "memory_cohort": "no paired 128K telemetry published" } }, { "label": "128K server smoke decode_rate", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L54", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 61, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K server smoke", "memory_cohort": "no paired 128K telemetry published" } }, { "label": "128K long-context needle prefill_rate", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L55", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 104805, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K long-context needle", "memory_cohort": "no paired 128K telemetry published" } }, { "label": "128K long-context needle decode_rate", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L55", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 104805, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K long-context needle", "memory_cohort": "no paired 128K telemetry published" } }, { "label": "32K telemetry startup", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "startup_duration", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false } }, { "label": "32K telemetry host_rss", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "peak_host_RSS", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false } }, { "label": "32K telemetry minimum_memavailable", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "minimum_MemAvailable_in_reported_monitor_scope", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false } }, { "label": "32K telemetry swap_use", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "maximum_swap_use", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false } }, { "label": "32K telemetry GPU_memory", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "peak_GPU_memory", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false } }, { "label": "canonical MTP n1 peak GPU memory", "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L61", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 1, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": null, "context_limit_tokens": 131072, "cache_state": null, "memory_cohort": "canonical MTP n1 short gates; not baseline or long-context peak", "memory_kind": "peak_GPU_memory" } } ], "deployment_options": [], "recipe_parameters": null, "measurement_scope": "source_request_latency_memory_and_gate_observations", "notes": [ "One source document can contain different experiments and versions; not an independent experiment count." ], "limitations": [ "No independent site model execution.", "Missing sampling, exact executed image digests and measurement boundaries remain null.", "Configured window, completed input, scheduler ceiling and measured full-window concurrency are separate quantities." ], "comparison_status": "not_comparable", "editorial_status": "source_derived_detailed_article", "public_benchmark_snapshot": false, "source": { "publisher": "kutaelee", "url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe", "repository": "kutaelee/qwen38-flash-next-5090-128k-runtime-recipe", "platform": "github", "revision": "fca75dd108ce04b6a79db3b45934ed58bba893bb", "document_path": "README.md", "pinned_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md", "report_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md", "document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "last_commit_at": "2026-09-18T12:12:22Z", "checked_on": "2026-10-18", "metadata_status": "checked" }, "supplemental_sources": [ { "document_id": "qwen5090-method", "parent_record_id": "qwen38-flash-next-5090-offload-gates", "publisher": "kutaelee", "repository": "kutaelee/qwen38-flash-next-5090-128k-runtime-recipe", "revision": "fca75dd108ce04b6a79db3b45934ed58bba893bb", "document_path": "docs/methodology.md", "pinned_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/docs/methodology.md", "document_sha256": "54d61a37de6873bcebe34c2ad34978a9b0a7191232fb865c364540cdd3e4c706", "checked_on": "2026-10-18" }, { "document_id": "qwen5090-limits", "parent_record_id": "qwen38-flash-next-5090-offload-gates", "publisher": "kutaelee", "repository": "kutaelee/qwen38-flash-next-5090-128k-runtime-recipe", "revision": "fca75dd108ce04b6a79db3b45934ed58bba893bb", "document_path": "docs/limitations.md", "pinned_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/docs/limitations.md", "document_sha256": "029f3cfc9a4b8e2e03eba92137e4df3ca722782b13af4dd990295a8fe1c270d2", "checked_on": "2026-10-18" }, { "document_id": "qwen5090-config", "parent_record_id": "qwen38-flash-next-5090-offload-gates", "publisher": "kutaelee", "repository": "kutaelee/qwen38-flash-next-5090-128k-runtime-recipe", "revision": "fca75dd108ce04b6a79db3b45934ed58bba893bb", "document_path": "configs/ik-llama-baseline-128k.example.sh", "pinned_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/configs/ik-llama-baseline-128k.example.sh", "document_sha256": "fc5de996c4e03b80de59f21597ae543a37ff829822641bf2117426b9c6f09f28", "checked_on": "2026-10-18" } ], "license": { "declared_spdx": "mit", "scope": "Source declaration; models, runtime and dependencies retain their exact terms." }, "editorial": { "version": 1, "published_on": "2026-10-18", "modified_on": "2026-10-18", "byline": "Run AI Here editorial", "creation_method": "Original source-based interpretation; local Spark Qwen draft with Codex factual review, no site model benchmark.", "answer": "The pinned MXFP4 GGUF runs through ik_llama.cpp merge d854dfd36a7262fc46cb7542716b04a9f6d4b47d on one RTX 5090 with WSL2, a 128 GiB-class host and 96 GiB VM ceiling. The baseline puts 38 expert layers on CPU, keeps mmap, disables the CUDA pinned-host override and uses Q8_0 KV. One request and one slot are used throughout; this is not full-model GPU residence.", "sections": [ { "heading": "Which memory placement belongs to the report?", "paragraphs": [ "The pinned MXFP4 GGUF runs through ik_llama.cpp merge d854dfd36a7262fc46cb7542716b04a9f6d4b47d on one RTX 5090 with WSL2, a 128 GiB-class host and 96 GiB VM ceiling. The baseline puts 38 expert layers on CPU, keeps mmap, disables the CUDA pinned-host override and uses Q8_0 KV. One request and one slot are used throughout; this is not full-model GPU residence." ], "citations": [ "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md", "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/configs/ik-llama-baseline-128k.example.sh" ] }, { "heading": "What input was actually completed?", "paragraphs": [ "The source reports a 26253-token needle under a 32768 window, a 61-token short smoke under a 131072 window, and a 104805-token exact needle under that longer window. Throughput comes from server timing. Completion lengths, needle depth, per-gate sampling and repeated-run dispersion are absent. A short smoke with a large configured ceiling is not a full-window input test." ], "citations": [ "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md", "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/docs/methodology.md" ] }, { "heading": "Which memory observations are paired with which gate?", "paragraphs": [ "Startup, peak RSS, minimum MemAvailable, swap and peak GPU memory are reported for the completed 32K monitoring gate. The longer 104805-token gate has no paired peak telemetry in these selected documents. The MemAvailable sampling boundary is unspecified, so its relationship to the WSL ceiling cannot be assumed. Keep original KiB and MiB labels and do not sum overlapping allocations." ], "citations": [ "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md", "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/docs/limitations.md" ] }, { "heading": "What happened to speculation?", "paragraphs": [ "MTP n=4 and canonical n=1 passed the tested correctness checks but failed speed promotion on this particular CPU-expert split. The canonical MTP short gates reached 31775 MiB peak GPU memory. Ngram speculative decoding was not run because its prerequisite failed; it has no measured benefit here. This is separate from the Q8 n-gram embedding table present in the artifact." ], "citations": [ "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md", "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/docs/limitations.md" ] } ], "facts": [], "reported_results": [ { "label": "32K needle gate prefill_rate", "metric": "prefill_rate", "value": 271.64, "unit": "tokens/s", "statistic": "one reported completed gate; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "workload": "32K needle gate", "memory_cohort": "completed 32K monitoring" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 53, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L53" }, { "label": "32K needle gate decode_rate", "metric": "decode_rate", "value": 16.53, "unit": "tokens/s", "statistic": "one reported completed gate; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "workload": "32K needle gate", "memory_cohort": "completed 32K monitoring" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 53, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L53" }, { "label": "128K server smoke prefill_rate", "metric": "prefill_rate", "value": 17.02, "unit": "tokens/s", "statistic": "one reported completed gate; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 61, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K server smoke", "memory_cohort": "no paired 128K telemetry published" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 54, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L54" }, { "label": "128K server smoke decode_rate", "metric": "decode_rate", "value": 15.26, "unit": "tokens/s", "statistic": "one reported completed gate; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 61, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K server smoke", "memory_cohort": "no paired 128K telemetry published" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 54, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L54" }, { "label": "128K long-context needle prefill_rate", "metric": "prefill_rate", "value": 245.72, "unit": "tokens/s", "statistic": "one reported completed gate; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 104805, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K long-context needle", "memory_cohort": "no paired 128K telemetry published" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 55, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L55" }, { "label": "128K long-context needle decode_rate", "metric": "decode_rate", "value": 13.94, "unit": "tokens/s", "statistic": "one reported completed gate; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 104805, "context_limit_tokens": 131072, "cache_state": null, "workload": "128K long-context needle", "memory_cohort": "no paired 128K telemetry published" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 55, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L55" }, { "label": "32K telemetry startup", "metric": "startup", "value": 25.15, "unit": "s", "statistic": "source monitored peak/minimum; no distribution", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "startup_duration", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 57, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57" }, { "label": "32K telemetry host_rss", "metric": "host_rss", "value": 53901956, "unit": "KiB", "statistic": "source monitored peak/minimum; no distribution", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "peak_host_RSS", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 57, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57" }, { "label": "32K telemetry minimum_memavailable", "metric": "minimum_memavailable", "value": 86701016, "unit": "KiB", "statistic": "source monitored peak/minimum; no distribution", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "minimum_MemAvailable_in_reported_monitor_scope", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 57, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57" }, { "label": "32K telemetry swap_use", "metric": "swap_use", "value": 63708, "unit": "KiB", "statistic": "source monitored peak/minimum; no distribution", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "maximum_swap_use", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 57, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57" }, { "label": "32K telemetry GPU_memory", "metric": "GPU_memory", "value": 25813, "unit": "MiB", "statistic": "source monitored peak/minimum; no distribution", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 0, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": 26253, "context_limit_tokens": 32768, "cache_state": null, "memory_cohort": "completed 32K gate only; not paired with the 104805-token gate", "memory_kind": "peak_GPU_memory", "MemAvailable_sampling_scope": null, "VM_ceiling_and_host_MemAvailable_same_scope_verified": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 57, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L57" }, { "label": "canonical MTP n1 peak GPU memory", "metric": "GPU_memory", "value": 31775, "unit": "MiB", "statistic": "maximum in completed MTP short gates", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "jamesrogers/Qwen3.8-Flash-Next-MTP-MXFP4-GGUF", "model_revision": "4a768bf4458a83e1c80667b4d818ec721080546e", "artifact_SHA256": "5a4236a6eeda73d23c151f8ed19e5457b0bf1935e1c02c1d42d98a0e26dadde2", "engine": "ik_llama.cpp", "runtime_execution_revision": "d854dfd36a7262fc46cb7542716b04a9f6d4b47d", "MTP_patch_head": "108dfaa77fdaa138c46b043bfff0e94fbe2adff7", "system": "single RTX 5090 32607 MiB; 16-core/32-thread desktop CPU; 128 GiB-class host; Ubuntu WSL2 with 96 GiB VM ceiling", "node_count": 1, "concurrency": 1, "configured_max_requests": 1, "weight_precision": "MXFP4 GGUF, Q8 n-gram table", "CPU_expert_layers": 38, "defer_experts": true, "mmap": true, "GGML_CUDA_NO_PINNED": true, "kv_dtype": "Q8_0", "draft_tokens": 1, "ngram_speculation": false, "ngram_embedding_table_present": true, "temperature": null, "top_p": null, "thinking": null, "output_tokens": null, "actual_output_tokens": null, "repetitions": null, "experiment_date": null, "throughput_origin": "server timing fields, not a client estimate", "input_tokens": null, "context_limit_tokens": 131072, "cache_state": null, "memory_cohort": "canonical MTP n1 short gates; not baseline or long-context peak", "memory_kind": "peak_GPU_memory" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 61, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L61" } ], "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0" }, "memory_observations": [ { "kind": "unpaired_memory_telemetry", "measured_context_limit_tokens": 32768, "measured_input_tokens": 26253, "longer_needle_input_tokens": 104805, "longer_needle_peak_RAM": null, "host_MemAvailable_scope": null, "CPU_expert_layers": 38, "source_GGUF_bytes": 126846153856 } ], "long_context_cases": [ { "classification": "source_reported_exact_needle_success", "actual_input_tokens": 26253, "configured_limit_tokens": 32768, "actual_output_tokens": null, "exact_match": true, "needle_depth_fraction": null, "general_reasoning_quality_proven": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 53, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L53" }, { "classification": "source_reported_short_smoke", "actual_input_tokens": 61, "configured_limit_tokens": 131072, "actual_output_tokens": null, "exact_match": true, "needle_depth_fraction": null, "general_reasoning_quality_proven": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 54, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L54" }, { "classification": "source_reported_exact_needle_success", "actual_input_tokens": 104805, "configured_limit_tokens": 131072, "actual_output_tokens": null, "exact_match": true, "needle_depth_fraction": null, "general_reasoning_quality_proven": false, "source_document_id": "qwen5090-readme", "source_document_sha256": "0668bd3d2e2ca6ed48b4ef13e9d9c56e4ce5c37c45ebbf8dc54e359bd9575de0", "source_line": 55, "evidence_url": "https://github.com/kutaelee/qwen38-flash-next-5090-128k-runtime-recipe/blob/fca75dd108ce04b6a79db3b45934ed58bba893bb/README.md#L55" } ] } ] }, { "id": "deepseek-1017-vq-followup", "title": "Read the dated VQ follow-up with its missing runtime results", "needs": [ "articles" ], "purpose": "Return the complete A12 interpretation including the October 18 bounded source recheck. The absent 1017 VQ serving measurements remain unknown.", "expression": "articles.articles.filter(a => a.id === 'A12')", "expected_output": [ { "id": "A12", "path": "/guides/deepseek-1017-quant-memory-context/", "title": "DeepSeek 1017: Quantization, Memory and Context Evidence", "summary": "Separate a short EXL3 deployment matrix, two near-million-token needle checks and a VQ conversion that has not measured 1017 serving speed.", "answer": "Read three evidence states separately: short-output EXL3 endpoint rates, two seven-token near-1M needle completions, and VQ file/loader inspection. None establishes general long-context quality or lets an older checkpoint speed become a 1017 result.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "x00byte", "url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "platform": "github", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "README.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/README.md", "report_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/README.md", "document_sha256": "839509cc2365ea030d94360454437e66165c1087182ce6ccb85d23d85d5bb66d", "last_commit_at": "2026-10-17T18:27:21Z", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "deepseek-1017-dual-spark-quick" }, { "publisher": "Elevatormusic", "url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "platform": "github", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-18/INDEX.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md", "report_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md", "document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "last_commit_at": "2026-10-18T05:40:29Z", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "deepseek-1017-million-token-needle" }, { "publisher": "Baekpica", "url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "platform": "huggingface", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "README.md", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md", "report_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md", "document_sha256": "b9f85bae000a8aa0e270555d17b1f9bd859a79bc3f5f2d7fde7eb00607cc990f", "last_commit_at": "2026-10-17T11:05:38.000Z", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "deepseek-1017-vq-artifact-inspection" } ], "supplemental_sources": [ { "document_id": "dsdual-config-manifest.json", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "config/manifest.json", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/manifest.json", "document_sha256": "6b3da5443112253502befe2a866983a31d188af234aef5e03865a724dbe9b65a", "checked_on": "2026-10-18" }, { "document_id": "dsdual-config-tensorfold-knobs.env.example", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "config/tensorfold-knobs.env.example", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/tensorfold-knobs.env.example", "document_sha256": "e26dc4761458be238d048c5a03b524993e7c632c27f8457527e3ba0030bc58c0", "checked_on": "2026-10-18" }, { "document_id": "dsdual-config-vllm-knobs.env.example", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "config/vllm-knobs.env.example", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/vllm-knobs.env.example", "document_sha256": "ee4eb1535b45d28c47fdd5b136e4b04268a7955d2ac9ca83174afeb35c0d62d8", "checked_on": "2026-10-18" }, { "document_id": "dsdual-docs-deployment.md", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "docs/deployment.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/deployment.md", "document_sha256": "72fb210773604c15560e73d6db8ada144b0e953354201e42b3b7aebd4c018799", "checked_on": "2026-10-18" }, { "document_id": "dsdual-docs-benchmark-method.md", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "docs/benchmark-method.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/benchmark-method.md", "document_sha256": "33d83fe761ea2eaafaadee937fa212ffcec19d69a2528614776ad7fb1139ba3b", "checked_on": "2026-10-18" }, { "document_id": "dsdual-results-README.md", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "results/README.md", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/README.md", "document_sha256": "380393cc1e86d175f21af1c216b613a2aad741ed90c851b7e8f491bed5231acf", "checked_on": "2026-10-18" }, { "document_id": "dsdual-results-tensorfold-quick.csv", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "results/tensorfold-quick.csv", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv", "document_sha256": "3005b3be00581a962f9a269211eef2d598a6358d83c41e565ee1549b28e03c33", "checked_on": "2026-10-18" }, { "document_id": "dsdual-results-vllm-quick.csv", "parent_record_id": "deepseek-1017-dual-spark-quick", "publisher": "x00byte", "repository": "x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark", "revision": "e85ed7ec3a403d8820a83e5aa33abe0985b58896", "document_path": "results/vllm-quick.csv", "pinned_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv", "document_sha256": "befbf211d4f143798e12053ec21291cc77e0893284ccdb8d5a0aa19ef4722c6e", "checked_on": "2026-10-18" }, { "document_id": "dsneedle-runs-2026-10-18-needle-1m.json", "parent_record_id": "deepseek-1017-million-token-needle", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-18/needle-1m.json", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json", "document_sha256": "9b81971c1378ab5b0bef466219c3a7aa7e2f78315bddbfc5bd0837cfec2fde40", "checked_on": "2026-10-18" }, { "document_id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "parent_record_id": "deepseek-1017-million-token-needle", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "runs/2026-10-18/needle-1m-final.json", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json", "document_sha256": "c476006b019d46c68a334ad3763f3e8e0f0532d01dfc6fb800c043c225709ab1", "checked_on": "2026-10-18" }, { "document_id": "dsneedle-docs-KNOWN-LIMITS.md", "parent_record_id": "deepseek-1017-million-token-needle", "publisher": "Elevatormusic", "repository": "Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark", "revision": "9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32", "document_path": "docs/KNOWN-LIMITS.md", "pinned_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/KNOWN-LIMITS.md", "document_sha256": "eeb6bc6f7a3fce20654f977581e9718b124e70fc01ab358efebefe9afd9dc942", "checked_on": "2026-10-18" }, { "document_id": "ds1017vq-RECIPE.md", "parent_record_id": "deepseek-1017-vq-artifact-inspection", "publisher": "Baekpica", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "RECIPE.md", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/RECIPE.md", "document_sha256": "1b66bfc44386bd6b708b1d4250cdd0dd825d2b45eee18c74d2bebfbff9e62486", "checked_on": "2026-10-18" }, { "document_id": "ds1017vq-SOURCE.md", "parent_record_id": "deepseek-1017-vq-artifact-inspection", "publisher": "Baekpica", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "SOURCE.md", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SOURCE.md", "document_sha256": "d039fb41a620380e352de9794271a6b2855e5d6931979d43695f4d3a236d443e", "checked_on": "2026-10-18" }, { "document_id": "ds1017vq-SHA256SUMS", "parent_record_id": "deepseek-1017-vq-artifact-inspection", "publisher": "Baekpica", "repository": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "document_path": "SHA256SUMS", "pinned_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SHA256SUMS", "document_sha256": "6b98a0a8923558f69ad53a9d60c29332b984a59f8109a6115c7c88e684763b19", "checked_on": "2026-10-18" } ], "facts": [], "reported_results": [ { "label": "tensorfold C1 thinking off", "metric": "decode_rate", "value": 58.05409336065285, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "TensorFold Python 0.6.0 patched", "runtime_execution_revision": "c4646171139ee8a3c38103eaa1699dad226ec12b", "overlay_source_revision": "2dcce0303451da2a59f80e6c7413da8b3b0f7a57", "executed_image_digest": null, "thinking": false, "reasoning_effort": null, "input_tokens": 520, "actual_output_tokens": 64, "batch_wall_seconds": 2.4011132870218717, "end_to_end_output_wall_tps": 26.65430254620761, "TTFT_seconds": 1.313945789996069 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-tensorfold-quick.csv", "source_document_sha256": "3005b3be00581a962f9a269211eef2d598a6358d83c41e565ee1549b28e03c33", "source_line": 2, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv#L2", "source_record_id": "deepseek-1017-dual-spark-quick" }, { "label": "tensorfold C1 thinking 75", "metric": "decode_rate", "value": 77.49652509362433, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "TensorFold Python 0.6.0 patched", "runtime_execution_revision": "c4646171139ee8a3c38103eaa1699dad226ec12b", "overlay_source_revision": "2dcce0303451da2a59f80e6c7413da8b3b0f7a57", "executed_image_digest": null, "thinking": true, "reasoning_effort": 75, "input_tokens": 546, "actual_output_tokens": 64, "batch_wall_seconds": 2.141774678020738, "end_to_end_output_wall_tps": 29.881761446140466, "TTFT_seconds": 1.326543439005036 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-tensorfold-quick.csv", "source_document_sha256": "3005b3be00581a962f9a269211eef2d598a6358d83c41e565ee1549b28e03c33", "source_line": 10, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv#L10", "source_record_id": "deepseek-1017-dual-spark-quick" }, { "label": "vllm C1 thinking off", "metric": "decode_rate", "value": 36.8753802051545, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "vLLM + EXL3 overlay", "runtime_execution_revision": null, "overlay_source_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "executed_image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "thinking": false, "reasoning_effort": null, "input_tokens": 520, "actual_output_tokens": 64, "batch_wall_seconds": 2.5652991679962724, "end_to_end_output_wall_tps": 24.948357212460998, "TTFT_seconds": 0.8555283379973844 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-vllm-quick.csv", "source_document_sha256": "befbf211d4f143798e12053ec21291cc77e0893284ccdb8d5a0aa19ef4722c6e", "source_line": 2, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv#L2", "source_record_id": "deepseek-1017-dual-spark-quick" }, { "label": "vllm C1 thinking 75", "metric": "decode_rate", "value": 41.447290817386964, "unit": "tokens/s", "statistic": "one pass; no run-to-run uncertainty estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_artifact": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "Engram_shard_SHA256": { "model-00047-of-00048.safetensors": "76a05106ece259be83d1bfea0dcb54f23377864ee60fd321690f35e6698a2af5", "model-00048-of-00048.safetensors": "9dc51433e195bc78b5ad846b68643cc0351d3b174e5e4309222d3b7402c83afd" }, "weight_precision": "EXL3 2.9bpw; 6-bit dense MTP handling in vLLM overlay", "system": "two DGX Spark GB10, 128 GB unified memory each, TP2, both available RoCE links", "node_count": 2, "parallelism": "TP2", "experiment_date": null, "context_limit_tokens": 262144, "configured_max_requests": 8, "draft_depth_cap": 3, "concurrency": 1, "input_lines": 27, "output_tokens": 64, "max_output_tokens": 64, "repetitions": 1, "cache_state": "startup and earlier generation checks; no per-case warmup; TensorFold session/prepared caches disabled", "temperature": null, "top_p": null, "kv_dtype": null, "measurement_denominator": "(completion_tokens - 1) / first-to-last streamed delta interval; excludes initial prefill", "quality_evaluation": false, "long_response_steady_state_measured": false, "native_image_routing_in_text_matrix": false, "engine": "vLLM + EXL3 overlay", "runtime_execution_revision": null, "overlay_source_revision": "6f7d1590ad49a2b8995188e45d7b9db31e677452", "executed_image_digest": "sha256:2f0cf3adc0f989c1d446be274df864eb799630175f604c3b22b71b7205971dce", "thinking": true, "reasoning_effort": 75, "input_tokens": 546, "actual_output_tokens": 64, "batch_wall_seconds": 2.4199117650277913, "end_to_end_output_wall_tps": 26.447245277666145, "TTFT_seconds": 0.8985815939959139 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsdual-results-vllm-quick.csv", "source_document_sha256": "befbf211d4f143798e12053ec21291cc77e0893284ccdb8d5a0aa19ef4722c6e", "source_line": 10, "evidence_url": "https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv#L10", "source_record_id": "deepseek-1017-dual-spark-quick" }, { "label": "fd2365042f56 near-1M first token", "metric": "ttft", "value": 830.4, "unit": "s", "statistic": "single prompt, no variance estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "fd2365042f56", "input_tokens": 1034753, "output_tokens": 7, "cached_tokens": 0, "cache_state": "zero prefix-cache hit tokens; not a cold OS page-cache guarantee", "experiment_date": "2026-10-18T01:51:54Z", "needle_found": true, "reasoning_tokens": 0, "total_seconds": 835.8 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-needle-1m.json", "source_document_sha256": "9b81971c1378ab5b0bef466219c3a7aa7e2f78315bddbfc5bd0837cfec2fde40", "source_line": 8, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json#L8", "source_record_id": "deepseek-1017-million-token-needle" }, { "label": "92aa710b533d near-1M first token", "metric": "ttft", "value": 817.7, "unit": "s", "statistic": "single prompt, no variance estimate", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cached_tokens": 0, "cache_state": "zero prefix-cache hit tokens; not a cold OS page-cache guarantee", "experiment_date": "2026-10-18T04:47:07Z", "needle_found": true, "reasoning_tokens": 0, "total_seconds": 825.9 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "source_document_sha256": "c476006b019d46c68a334ad3763f3e8e0f0532d01dfc6fb800c043c225709ab1", "source_line": 8, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json#L8", "source_record_id": "deepseek-1017-million-token-needle" }, { "label": "near-1M final head minimum MemAvailable", "metric": "minimum_memavailable", "value": 1779, "unit": "MiB", "statistic": "minimum observed in this one fill, not a safe capacity guarantee", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cache_state": "same final needle; cached_tokens 0", "experiment_date": "2026-10-18", "memory_kind": "minimum_host_MemAvailable_during_prefill", "node": "head", "earlyoom_limit_MiB_at_measurement": 1536, "earlyoom_later_limit_MiB": 1024, "OS_memory_controls_changed": true, "ordinary_host_baseline_inferred": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-INDEX.md", "source_document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "source_line": 59, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L59", "source_record_id": "deepseek-1017-million-token-needle" }, { "label": "near-1M final worker minimum MemAvailable", "metric": "minimum_memavailable", "value": 2542, "unit": "MiB", "statistic": "minimum observed in this one fill, not a safe capacity guarantee", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "model_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "weight_precision": "EXL3 2.9bpw", "system": "two DGX Spark GB10, TP2", "node_count": 2, "engine": "TensorFold Python 0.6.6", "runtime_execution_revision": null, "executed_image_digest": null, "context_limit_tokens": 1048576, "concurrency": 1, "needle_depth_fraction": 0.63, "workload": "one exact-string needle in random filler lines", "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "general_quality_evaluation": false, "multi_request_long_context_validated": false, "runtime_revision_prefix": "92aa710b533d", "input_tokens": 1035020, "output_tokens": 7, "cache_state": "same final needle; cached_tokens 0", "experiment_date": "2026-10-18", "memory_kind": "minimum_host_MemAvailable_during_prefill", "node": "worker", "earlyoom_limit_MiB_at_measurement": 1536, "earlyoom_later_limit_MiB": 1024, "OS_memory_controls_changed": true, "ordinary_host_baseline_inferred": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "dsneedle-runs-2026-10-18-INDEX.md", "source_document_sha256": "e69d8c270e6407ddbaacb82da36cc48994b6f20fd4300d176753371b4e23fb8e", "source_line": 59, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md#L59", "source_record_id": "deepseek-1017-million-token-needle" }, { "label": "1017 VQ base GGUF", "metric": "artifact_size", "value": 113556639424, "unit": "bytes", "statistic": "source-reported file bytes; not a RAM requirement", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "model_revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "base_model_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "quantizer_revision": "7fb26cc332d740416792204a29c7d13d2986caf5", "vision_converter_revision": "e4ea07889144d85acd33b2490ec4f44032644fb0", "quantization": "YoungAi vq8sh14-q4k-mtpnative; routed-expert VQ, q4_K skeleton; native MTP towers", "measurement_kind": "publisher artifact layout inspection, not runtime evaluation", "experiment_date": "2026-10-17 status in card", "system": null, "engine": "target ds4-dfm-rs; loader check passed, serving unmeasured on 1017", "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "context_limit_tokens": 1048576, "context_limit_origin": "model metadata, not measured serving result", "speed_1017_measured": false, "quality_1017_measured": false, "peak_runtime_memory_measured": false, "base_model_speed_inherited": false, "filename": "DeepSeek-V4.1-Flash-vq8sh14-q4k-mtpnative.gguf", "memory_kind": "artifact_disk_file_bytes", "Engram_included": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "ds1017vq-readme", "source_document_sha256": "b9f85bae000a8aa0e270555d17b1f9bd859a79bc3f5f2d7fde7eb00607cc990f", "source_line": 97, "evidence_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md#L97", "source_record_id": "deepseek-1017-vq-artifact-inspection" }, { "label": "1017 BF16 vision sidecar", "metric": "artifact_size", "value": 971066016, "unit": "bytes", "statistic": "source-reported file bytes; not a RAM requirement", "uncertainty": null, "conditions": { "model": "DeepSeek-V4.1-Flash-1017", "model_artifact": "Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF", "model_revision": "f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5", "base_model_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "quantizer_revision": "7fb26cc332d740416792204a29c7d13d2986caf5", "vision_converter_revision": "e4ea07889144d85acd33b2490ec4f44032644fb0", "quantization": "YoungAi vq8sh14-q4k-mtpnative; routed-expert VQ, q4_K skeleton; native MTP towers", "measurement_kind": "publisher artifact layout inspection, not runtime evaluation", "experiment_date": "2026-10-17 status in card", "system": null, "engine": "target ds4-dfm-rs; loader check passed, serving unmeasured on 1017", "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "context_limit_tokens": 1048576, "context_limit_origin": "model metadata, not measured serving result", "speed_1017_measured": false, "quality_1017_measured": false, "peak_runtime_memory_measured": false, "base_model_speed_inherited": false, "filename": "DeepSeek-V4.1-Flash.vision.gguf", "memory_kind": "artifact_disk_file_bytes", "Engram_included": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "ds1017vq-readme", "source_document_sha256": "b9f85bae000a8aa0e270555d17b1f9bd859a79bc3f5f2d7fde7eb00607cc990f", "source_line": 98, "evidence_url": "https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md#L98", "source_record_id": "deepseek-1017-vq-artifact-inspection" } ], "conditions": { "one_needle_is_general_long_context_quality": false, "EXL3_and_VQ_results_interchangeable": false, "VQ_1017_speed_measured": false, "VQ_1017_quality_measured": false, "independent_site_runs": 0 }, "deployment_profiles": [ { "id": "dsdual-python-profile", "model_artifact": "LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw", "checkpoint_revision": "afc7aa0aab98ee735c0f54e0a122019af1ac08f1", "Engram_revision": "1bae0e1088993b9b82de08f59de5be5a34e42170", "engine": "TensorFold Python 0.6.0 patched", "runtime_revision": "c4646171139ee8a3c38103eaa1699dad226ec12b", "context_limit_tokens": 262144, "configured_shared_pool_tokens": 524288, "configured_request_slots": 8, "configured_MemAvailable_floor_GiB": 5, "configured_hard_floor_GiB": 4, "memory_floors_are_measured_peaks": false, "session_cache": false, "prepared_cache_writes": false, "source_current_recipe_has_native_image_routing": true, "text_matrix_predates_image_flag": true } ], "memory_observations": [ { "record_id": "deepseek-1017-million-token-needle", "kind": "minimum_host_MemAvailable_during_final_fill", "head_MiB": 1779, "worker_MiB": 2542, "host_tuning_required_in_source": true, "safe_capacity_guarantee": false }, { "record_id": "deepseek-1017-vq-artifact-inspection", "kind": "file_inventory_not_RAM", "base_bytes": 113556639424, "vision_bytes": 971066016, "Engram_shard_bytes": [ 101535150936, 101537926640 ], "actual_peak_RAM": null, "speed_and_quality_1017": null } ], "generation": { "text_draft": "local Spark Qwen if available, then original source review and revision by Codex; actual per-item receipt in editorial-batch5 report", "review": "Run AI Here with Codex", "image": "matplotlib from original reported observations; no generated measured data", "local_first": true }, "image": { "url": "/assets/deepseek-1017-quant-memory-context-source.webp", "alt": "DeepSeek 1017 short-run EXL3 decode, separate near-million-token needle latency and a VQ artifact-size receipt with unknown runtime performance.", "width": 1600, "height": 900, "sha256": "5a98707bcce31245a241a8cecfb66e03d46d09d9826de8f6d4d79854874eeac5", "caption": "Three distinct source states. The short EXL3 decode runs use 520 input and 64 output tokens; each near-1M test returns seven tokens at one needle depth. VQ file bytes and loader binding do not establish 1017 serving speed or runtime memory." }, "long_context_cases": [ { "id": "dsneedle-runs-2026-10-18-needle-1m.json", "classification": "source_reported_single_needle_success", "configured_limit_tokens": 1048576, "actual_input_tokens": 1034753, "actual_output_tokens": 7, "needle_depth_fraction": 0.63, "exact_match": true, "TTFT_seconds": 830.4, "cached_tokens": 0, "runtime_revision_prefix": "fd2365042f56", "executed_image_digest": null, "sampling": null, "quality_scope": "one needle at one depth; not broad reasoning, multi-needle, long-response or concurrency validation", "source_document_id": "dsneedle-runs-2026-10-18-needle-1m.json", "source_document_sha256": "9b81971c1378ab5b0bef466219c3a7aa7e2f78315bddbfc5bd0837cfec2fde40", "source_line": 3, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json#L3" }, { "id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "classification": "source_reported_single_needle_success", "configured_limit_tokens": 1048576, "actual_input_tokens": 1035020, "actual_output_tokens": 7, "needle_depth_fraction": 0.63, "exact_match": true, "TTFT_seconds": 817.7, "cached_tokens": 0, "runtime_revision_prefix": "92aa710b533d", "executed_image_digest": null, "sampling": null, "quality_scope": "one needle at one depth; not broad reasoning, multi-needle, long-response or concurrency validation", "source_document_id": "dsneedle-runs-2026-10-18-needle-1m-final.json", "source_document_sha256": "c476006b019d46c68a334ad3763f3e8e0f0532d01dfc6fb800c043c225709ab1", "source_line": 3, "evidence_url": "https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json#L3" } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# DeepSeek 1017: Quantization, Memory and Context Evidence\n\nDeepSeek 1017 has several usable community evidence trails, but they answer different questions. A dual-Spark EXL3 matrix measures short endpoint requests. Two near-million-token runs retrieve one string and return seven tokens. A VQ conversion reports file layout and a loader check while explicitly leaving 1017 serving speed unmeasured. Keep those states separate before choosing a quantization or budgeting memory.\n\n![DeepSeek 1017 short-run EXL3 decode, separate near-million-token needle latency and a VQ artifact-size receipt with unknown runtime performance.](/assets/deepseek-1017-quant-memory-context-source.webp)\n\n*The chart preserves three source states. Short decode rates are not near-1M generation rates, and VQ file bytes are not a runtime RAM requirement. No independent site model benchmark was performed.*\n\n## Identify the EXL3 model and matching Engrams\n\nThe [dual-Spark deployment](https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/README.md) uses `LibertAIDAI/DeepSeek-V4.1-Flash-1017-EXL3-2.9bpw` at `afc7aa0aab98ee735c0f54e0a122019af1ac08f1`. Its [manifest](https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/manifest.json) separately pins official Engram tables to `deepseek-ai/DeepSeek-V4.1-Flash-1017` at `1bae0e1088993b9b82de08f59de5be5a34e42170`. A document commit, converted-weight commit and Engram commit identify different objects.\n\nThis deployment uses two 128 GB GB10 systems and TP2 over RoCE, with checkpoint reads through a read-only NFS export and rank-local Engram lookup. The TensorFold path is patched Python 0.6.0; the vLLM arm uses an EXL3 overlay. These are concrete integrations, not an instruction that an arbitrary current installation of either engine accepts the files unchanged.\n\n## What do the short-run rates actually measure?\n\nThe [method](https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/benchmark-method.md) uses deterministic ledger prompts: 520 input tokens with thinking disabled and 546 at reasoning effort 75. Selected C1 runs produce 64 output tokens. Each is one pass; no run-to-run variance estimate is available.\n\n| C1 arm | Thinking off | Effort 75 |\n|---|---:|---:|\n| TensorFold Python 0.6.0 | 58.1 tokens/s | 77.5 tokens/s |\n| vLLM with EXL3 overlay | 36.9 tokens/s | 41.4 tokens/s |\n\nThe table rounds the [TensorFold CSV](https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/tensorfold-quick.csv) and [vLLM CSV](https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/results/vllm-quick.csv) to one decimal; JSON retains the original numeric precision. Decode divides completion tokens after the first by the first-to-last streamed-delta interval. Initial prefill is excluded. Batch wall throughput is a separate quantity and must not be replaced by this decode rate.\n\nAn effort-75 response capped at 64 tokens can finish its budget inside thinking. A faster short interval therefore does not establish better answers or sustained long-generation capacity. The text matrix also predates the explicit native-image-routing flag. A later red-image smoke check in the deployment report is a serving-path check, not an image-understanding benchmark.\n\n## A shared pool is an admission setting\n\nThe [TensorFold deployment description](https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/docs/deployment.md) reports eight slots, a 524,288-token shared pool and a 262,144-token per-request ceiling. Its [settings](https://github.com/x00byte/DeepSeek-V4.1-Flash-1017-Dual-Spark/blob/e85ed7ec3a403d8820a83e5aa33abe0985b58896/config/tensorfold-knobs.env.example) configure 5 GiB and 4 GiB memory floors, full prefill, and disabled session/prepared-cache writes.\n\nThese are this profile's controls, not peak RAM measurements. They do not demonstrate eight simultaneous maximum-length requests. They also do not govern the separate near-1M experiment below. Comparing its lower free-memory reading with these floors would incorrectly combine different builds, launch settings and memory policies.\n\n## Read the near-1M result as one specific retrieval task\n\nThe [18 October run index](https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/INDEX.md) and two saved JSON records describe one exact-string needle at 63 percent depth in random filler. Both runs found their requested string, with zero cached prompt tokens and seven completion tokens.\n\n| Saved run | Actual input tokens | Output tokens | First token |\n|---|---:|---:|---:|\n| Earlier build, `fd2365042f56` | 1,034,753 | 7 | 830.4 s |\n| Final setup, `92aa710b533d` | 1,035,020 | 7 | 817.7 s |\n\nThe [earlier record](https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m.json) and [final record](https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/runs/2026-10-18/needle-1m-final.json) retain exact counts and timestamps. These are different build/setup observations, not two unchanged-configuration repetitions. Runtime identifiers are reported prefixes; exact image digests and sampling values remain unknown. Zero reasoning tokens in the output is preserved without inventing an unlogged thinking flag.\n\nThe result supports a narrow conclusion: each named setup accepted that prompt and found that string. It does not establish broad reasoning over a million tokens, multiple-needle recall, long-answer stability, multimodal context or concurrent full-window requests. The seven-token completion matters as much as the large input count.\n\n## Keep the small memory margin attached to the result\n\nDuring the final fill, minimum host `MemAvailable` was 1,779 MiB on the head and 2,542 MiB on the worker. These are observed minimum available-memory readings, not the model's allocated memory or a recommended spare-RAM budget.\n\nThe source describes swap restrictions, host reclaim/compaction settings and Engram page dropping. Its earlyoom threshold was 1,536 MiB during the reported run and was later changed to 1,024 MiB. This is a tuned, narrow-margin case. The [known limits](https://github.com/Elevatormusic/DeepSeek-V4.1-Flash-1017-EXL3-TensorFold-2x-DGX-Spark/blob/9c6ab02340b770c6c3e1a9a4d5e70a56c5829d32/docs/KNOWN-LIMITS.md) and run index also retain slower prefill states and later final-image runs below earlier results. The site does not turn those whole-host changes into general deployment defaults.\n\n## VQ conversion: file inspection before speed claims\n\nThe [1017 VQ artifact card](https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md) reports a 113,556,639,424-byte base and a 971,066,016-byte vision sidecar. The base excludes the two Engram files. A 1,000-tensor loader binding check passed, but this snapshot explicitly has no 1017-specific serving speed, peak runtime RAM or model-quality result.\n\nIts [recipe](https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/RECIPE.md) pins the quantizer and vision converter separately. Its [provenance](https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SOURCE.md) explains why matching names and sizes are insufficient: 1017 Engram and vision contents differ from older V4.1 files. Preserve revision and the supplied [SHA-256 identities](https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/SHA256SUMS), including when a loader accepts an older file without error.\n\nThe native 1,048,576-token metadata field is not a measured VQ serving result. Older V4.1 speed/fidelity examples in the card remain older-checkpoint examples. They cannot fill the missing 1017 values, and file bytes cannot account for all runtime state, buffers, cache residency or service overhead.\n\n## Use the evidence without extending it\n\nThe [previous TensorFold explanation](/guides/deepseek-1017-tensorfold-community/) discusses other named configurations; it does not make their settings interchangeable with these cases. For AI access, `deepseek-1017-long-context` preserves both narrow retrieval cases and headroom, while `deepseek-1017-quantization` returns the short EXL3 report beside the VQ inspection. See [For AI](/for-ai/) for executable examples.\n\nChoose the evidence type needed for the next decision: verified file identity, endpoint behaviour, memory observation or task correctness. Keep actual input/output tokens, cache state, runtime and weight identities beside every result. Where the source supplies no execution result, the corresponding field stays null rather than becoming zero or an inherited benchmark.\n\n## VQ follow-up checked on October 18\n\nThe publisher card was rechecked at revision `f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5`. It still explicitly leaves 1017-specific GB10 speed and quality unmeasured. The selected [runtime guide](https://github.com/Baekpica/ds4-dfm-rs/blob/c9425237669057937178e0598b3114ed9f8a15ec/docs/deepseek41-flash.md) and [performance document](https://github.com/Baekpica/ds4-dfm-rs/blob/c9425237669057937178e0598b3114ed9f8a15ec/docs/performance.md) do not supply a 1017 VQ run that fills those gaps. The Hub discussion listing was empty at this check. This is a review of those named sources, not proof that no report exists anywhere. File identity remains useful; old V4.1 speed, quality and residency figures remain outside the 1017 measurement set. No new VQ serving result is added.\n\n## Further VQ source check on October 18\n\nA later bounded check retains the same [1017 VQ card revision](https://huggingface.co/Baekpica/DeepSeek-V4.1-Flash-1017-VQ-GGUF/blob/f81ee2e9801fcc58fd51ae75dd827467ffc0b9e5/README.md). The runtime has advanced to `9feb2d79fd0f84066d5c06ace4f4423e49fbda99`, but its [model guide](https://github.com/Baekpica/ds4-dfm-rs/blob/9feb2d79fd0f84066d5c06ace4f4423e49fbda99/docs/deepseek41-flash.md) and [performance document](https://github.com/Baekpica/ds4-dfm-rs/blob/9feb2d79fd0f84066d5c06ace4f4423e49fbda99/docs/performance.md) still contain no 1017-specific report. Hub discussions remain empty in this snapshot. No new 1017 VQ throughput, quality score or peak runtime RAM can be added from these checked sources; those fields remain null. Older V4.1 performance and the unrelated EXL3 results are not transferred. This describes the named-source review, not an exhaustive absence claim across the community.\n" } ] }, { "id": "qwen-long-concurrency-failures", "title": "Read Qwen long concurrent batches, memory scopes and failures", "needs": [ "articles" ], "purpose": "Return A15 complete, preserving original source identities, concurrent batch conditions, campaign-only peak and open-PR failure scope.", "expression": "articles.articles.filter(a => a.id === 'A15')", "expected_output": [ { "id": "A15", "path": "/guides/qwen-flash-next-long-context-concurrency/", "title": "Qwen Long Context: Concurrency, Memory and Failure Evidence", "summary": "Read 128K and 490K Qwen batches, a four-GPU 128K matrix, sampled peak VRAM and a Mamba crash with original versions and measurement limits.", "answer": "Four submitted requests do not guarantee four running requests. The sources contain completed long-context batches, but their cache states, timing denominators and memory windows differ. Keep the four-GPU campaign peak separate from per-cell latency, and retain the original scheduler failure.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "platform": "github", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "RESULTS.md", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md", "document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "last_commit_at": "2026-10-17T16:41:23Z", "url": "https://github.com/jpezzulli/sglang-rtxpro6000", "report_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "qwen38-flash-next-rtxpro-long-concurrency" } ], "supplemental_sources": [ { "document_id": "penny-readme", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "README.md", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/README.md", "document_sha256": "7645e7297eb1326393e8e4106fb73334980dfe44540b45e51655bc1dadd295ad", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "penny-community-results", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "COMMUNITY-RESULTS.md", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/COMMUNITY-RESULTS.md", "document_sha256": "2bbe9428f0701dc597dd0417193c12e72157dff2aca3ee6481b3104755a1301c", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "penny-release-3.0", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "release-3.0.json", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/release-3.0.json", "document_sha256": "1799eca8496bf143fc48bbdc83a4f6ccc34b8773c43083b47e29721b1ceb3dcf", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "penny-release-3.0.2", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "release-3.0.2.json", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/release-3.0.2.json", "document_sha256": "4843d621fa7f735a4c95d6bc79c9582d465cfaecef6b473689158d8c190920e2", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "qwen-cstech-4gpu", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "CS Technology Services", "repository": null, "revision": "snapshot-2026-10-18", "document_path": "qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell", "pinned_url": "https://www.cstech.dev/articles/qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell", "document_sha256": "b81878ba45f2f6b7a6de06b0bf930bd1dfa3cb497202ed3fcd368efe66065930", "last_commit_at": "2026-08-26", "checked_on": "2026-10-18", "public_URL_is_immutable": false, "revision_kind": "Mutable original report; retrieval snapshot date and SHA256, no Git revision" }, { "document_id": "qwen-mamba-incident", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "leehom0123", "repository": "sgl-project/sglang", "revision": "comment-5606493113", "document_path": "/issues/37619/comments/5606493113", "pinned_url": "https://github.com/sgl-project/sglang/pull/37619#issuecomment-5606493113", "document_sha256": "d36cc21fa7b9577474ba20c9fe26becbb3c61bab9b9c4a5c421447b9d905c3f4", "last_commit_at": "2026-09-09T18:05:35Z", "checked_on": "2026-10-18", "public_URL_is_immutable": false, "revision_kind": "GitHub issue comment ID; mutable body with saved SHA256" } ], "facts": [], "reported_results": [ { "label": "128K C4 option off", "metric": "aggregate_output_rate", "value": 367.127, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K four-request three-key retrieval", "input_length_source_label": "128K", "concurrency": 4, "precision_option": "option off", "actual_output_tokens_group": 15725, "group_wall_seconds": 42.833, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": null, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 231, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L231", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "128K C4 online FP8", "metric": "aggregate_output_rate", "value": 422.037, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K four-request three-key retrieval", "input_length_source_label": "128K", "concurrency": 4, "precision_option": "online FP8", "actual_output_tokens_group": 15898, "group_wall_seconds": 37.67, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": 30.152, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 231, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L231", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "490K C4 option off", "metric": "aggregate_output_rate", "value": 236.678, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K four-request three-key retrieval", "input_length_source_label": "490K", "concurrency": 4, "precision_option": "option off", "actual_output_tokens_group": 15455, "group_wall_seconds": 65.3, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": null, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 232, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L232", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "490K C4 online FP8", "metric": "aggregate_output_rate", "value": 329.01, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K four-request three-key retrieval", "input_length_source_label": "490K", "concurrency": 4, "precision_option": "online FP8", "actual_output_tokens_group": 15127, "group_wall_seconds": 45.977, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": 24.85, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 232, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L232", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "128K online FP8 C1 decode_rate", "metric": "decode_rate", "value": 195.634, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "128K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 229, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L229", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "128K online FP8 C1 ttft", "metric": "ttft", "value": 12.906, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "128K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 229, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L229", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "490K online FP8 C1 decode_rate", "metric": "decode_rate", "value": 172.644, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "490K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 230, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L230", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "490K online FP8 C1 ttft", "metric": "ttft", "value": 63.296, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "490K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 230, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L230", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "Post-graph free VRAM", "metric": "available_GPU_memory", "value": 7.52, "unit": "GiB", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "concurrency": null, "memory_kind": "post_graph_available_not_peak_used", "memory_cohort": "startup", "paired_long_text_batch_peak": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 243, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L243", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "Media-window minimum free VRAM", "metric": "available_GPU_memory", "value": 1897, "unit": "MiB", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "concurrency": null, "memory_kind": "sampled_minimum_free_not_per_long_text_batch_peak", "memory_cohort": "ten selected GPU-media checks; ordinary/large JPEG, concurrent images and video history", "paired_long_text_batch_peak": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 263, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L263", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "3.0 cold 64K prefill_rate", "metric": "prefill_rate", "value": 17235.9, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 63921, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "native mean input rate" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 44, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L44", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "3.0 cold 64K ttft", "metric": "ttft", "value": 4.71, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 63921, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "client first-token latency" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 44, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L44", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "3.0 cold 490K prefill_rate", "metric": "prefill_rate", "value": 8839.75, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 489915, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "native mean input rate" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 45, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L45", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "3.0 cold 490K ttft", "metric": "ttft", "value": 60.37, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 489915, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "client first-token latency" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 45, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L45", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "3.0 short C1 native mean", "metric": "native_decode_rate", "value": 218.63, "unit": "tokens/s", "statistic": "native mean; three regular repeats, no SD", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": 0.8, "top_p": 0.95, "thinking": "xhigh", "repetitions": 3, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 128, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "regular warmed screens", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "top_k": 40, "min_p": 0.05, "metric_denominator": "elapsed-weighted complete active native log intervals; excludes idle/prefill crossing and partial tail; aggregate if C4" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 26, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L26", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" }, { "label": "3.0 short C4 native mean", "metric": "native_decode_rate", "value": 489.35, "unit": "tokens/s", "statistic": "native mean; three regular repeats, no SD", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": 0.8, "top_p": 0.95, "thinking": "xhigh", "repetitions": 3, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 128, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "regular warmed screens", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 4, "top_k": 40, "min_p": 0.05, "metric_denominator": "elapsed-weighted complete active native log intervals; excludes idle/prefill crossing and partial tail; aggregate if C4" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 27, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L27", "source_record_id": "qwen38-flash-next-rtxpro-long-concurrency" } ], "conditions": { "independent_site_runs": 0, "controlled_cross_version_speedup": false, "per_cell_peak_RAM_known": false }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch7 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch7-qwen-long-hero.webp", "alt": "Penny Royal original tests, a four-GPU matrix, and a dated scheduler incident.", "width": 1600, "height": 900, "sha256": "c9c9a6a8d6a514b46d6c76d9fdb80b723964002b6f08613991356ea4aacb83d9", "caption": "Original reports only. No new Run AI Here model execution." }, "case_reports": [ { "id": "qwen-cstech-128k-matrix", "classification": "source_reported_duration_matrix", "source_url": "https://www.cstech.dev/articles/qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell", "source_snapshot_sha256": "b81878ba45f2f6b7a6de06b0bf930bd1dfa3cb497202ed3fcd368efe66065930", "public_URL_is_immutable": false, "source_published_on": "2026-08-26", "model": "Qwen/Qwen3.8-Flash-Next-FP8", "model_revision": null, "engine": "vLLM 0.1.dev20073+g8e685d198", "executed_image_digest": null, "runtime_execution_revision": null, "harness": "llm-inference-bench v0.4.29", "system": "4 RTX PRO 6000 Blackwell Max-Q, 300 W/card, Threadripper, PCIe Gen4, no NVLink", "parallelism": "TP4", "context_limit_tokens": 262144, "KV_pool_tokens": 2662752, "configured_max_requests": 256, "draft_tokens": 3, "temperature": null, "top_p": null, "thinking": null, "input_length_source_label": "128k nominal matrix context; exact per-cell prompt token counts unreported", "duration_seconds_per_cell": 30, "output_tokens_cap": 8192, "ignore_eos": true, "effective_concurrency_validated_by_source": true, "cache_state": "duration-mode integrated scout warms shared prefix; matrix-start TTFT includes cache hit and queueing", "cells": [ { "concurrency": 1, "aggregate_output_tokens_per_second": 128.5, "TTFT_seconds": 1.41 }, { "concurrency": 8, "aggregate_output_tokens_per_second": 505, "TTFT_seconds": 2.02 }, { "concurrency": 16, "aggregate_output_tokens_per_second": 729.9, "TTFT_seconds": 3.42 } ], "omitted_cell": { "concurrency": 32, "status": "not_run_capacity", "aggregate_output_tokens_per_second": null, "observed_runtime_crash": false }, "memory": { "kind": "sampled_peak_summed_GPU_used_over_entire_matrix", "value": 374976, "unit": "MiB", "sample_count": 501, "campaign_seconds": 1206, "paired_per_cell_peak": null, "host_peak_RAM": null }, "cold_C1": { "actual_input_tokens": 128021, "TTFT_seconds": 24.25, "prefill_tokens_per_second": 5279 }, "per_user_table_is_derived_aggregate_divided_by_streams": true }, { "id": "qwen-mamba-long-prefill-crash", "classification": "source_reported_scheduler_assertion", "source_url": "https://github.com/sgl-project/sglang/pull/37619#issuecomment-5606493113", "source_snapshot_sha256": "d36cc21fa7b9577474ba20c9fe26becbb3c61bab9b9c4a5c421447b9d905c3f4", "source_updated_at": "2026-09-09T18:05:35Z", "public_URL_is_immutable": false, "model": "Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "engine": "SGLang 0.0.0.dev1+g4ccff141d", "runtime_execution_revision": null, "executed_image_digest": null, "weight_precision": "modelopt_mixed", "kv_dtype": "fp8_e4m3", "context_length_source_label": "262k", "actual_prompt_tokens": null, "prompt_length_source_label": "approximately 140k", "chunked_prefill_size": 4096, "mamba_track_interval": 64, "mamba_slots": 51, "configured_max_requests": 12, "observed_running_requests": 2, "observed_queued_requests_reporter": 1, "queued_requests_at_cited_log_line": 0, "mamba_usage_before": 0.18, "mamba_usage_at_failure": 1, "full_token_usage_at_failure": 0.46, "HiCache": true, "hicache_ratio": 2, "write_policy": "write_through", "speculation": false, "error": "AssertionError: Can not alloc mamba cache", "outcome": "scheduler child exited; container auto-restarted, service outage during reload", "fix_reference": "https://github.com/sgl-project/sglang/pull/37619", "fix_PR_state": "open", "fix_PR_merged": false, "fix_PR_head_revision": "cd530ac217ac0a67b913f7083679c0890670d228", "author_GPU_reproducer_rerun": false, "universal_fix_verified": false, "peak_RAM": null } ], "memory_observations": [ { "kind": "sampled_peak_summed_GPU_used_over_entire_matrix", "value": 374976, "unit": "MiB", "sample_count": 501, "campaign_seconds": 1206, "paired_per_cell_peak": null, "host_peak_RAM": null }, { "kind": "v2.5.0_GPU_media_window_minimum_free", "value": 1897, "unit": "MiB", "long_text_C4_peak": null, "source_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md" } ], "long_context_cases": [ { "classification": "source_reported_four_submitted_stress", "recipe_release": "v2.5.0", "contributor": "BESTIA / Prometeus field report; separate contributor from runtime author", "source_url": "https://www.reddit.com/r/BlackwellPerformance/comments/1weaxdz/comment/p9i76fb/", "source_summary_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/COMMUNITY-RESULTS.md", "context_limit_tokens": 524288, "KV_pool_tokens": 1114304, "input_tokens": null, "input_length_source_label": "approximately 240K", "submitted_concurrency": 4, "simultaneous_running_concurrency": null, "rounds": 10, "total_requests": 40, "output_tokens_reported": 8192, "round_seconds_range": [ 93, 107 ], "HTTP_200": 40, "OOMs_reported": 0, "retractions_reported": 0, "aborts_reported": 0, "peak_pool_occupancy_percent": 57, "peak_memory_bytes": null, "queuing_observed": true, "retrieval_correctness": null } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# Qwen Long Context: Concurrency, Memory and Failure Evidence\n\n![Penny Royal original tests, a four-GPU matrix, and a dated scheduler incident.](/assets/batch7-qwen-long-hero.webp)\n\n*Original reports only. No new Run AI Here model execution.*\n\nLong context changes the question a deployment report needs to answer. A server can accept a large window, submit several requests and report a high token rate without keeping every long document resident at once. Useful evidence records the actual request shape, overlapping execution, queue, prefix cache, completion length and memory sampling window together. This note reads original Qwen3.8-Flash-Next reports with those distinctions intact. Run AI Here has not repeated their model tests.\n\n## One-card long batches: what actually overlapped\n\nThe [Penny Royal original results](https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md) contain September 11 v2.5.0 observations on one RTX PRO 6000 Blackwell, TP1. The relevant configuration uses RadixArk ModelOpt NVFP4, RAM PLE, FP8 KV, FR-Spec/native NEXTN, a 524,288-token request limit, an 824,384-token pool, 24 Mamba slots and 32 GiB HiCache/NIXL. “Online FP8” changes selected projections; it does not mean every model component changes precision.\n\n| Source-labelled workload | Option off, aggregate tokens/s | Online FP8, aggregate tokens/s | Online-arm output tokens / group seconds |\n|---|---:|---:|---|\n| 128K, four requests | 367.127 | 422.037 | 15,898 / 37.670 |\n| 490K, four requests | 236.678 | 329.010 | 15,127 / 45.977 |\n\nEach value is one batch observation. Aggregate means all output tokens divided by synchronized group makespan, including first-token waiting. Requests end at natural stop or a 4,096-token cap, so the arms produce different totals. These are contextual batch rates, with no repeated-run dispersion. The online arm records four-way overlap for 30.152 seconds at 128K and 24.850 seconds at 490K. The source does not establish four fully distinct, full-length resident prefix copies. Exact per-request input counts and cache sharing are missing in this summary.\n\n![Penny Royal v2.5.0 aggregate rates at 128K and 490K, with distinct off and online-FP8 arms.](/assets/batch7-qwen-long-rates.webp)\n\n*One batch per cell. These aggregate rates include synchronized group wall time and different completion totals.*\n\nThe matching online-FP8 C1 observations use exactly 1,024 output tokens: 195.634 tokens/s after the first token at 128K, and 172.644 at 490K. TTFT is 12.906 and 63.296 seconds respectively. Those long observations are single samples. The source explicitly says cold TTFT did not improve against its option-off samples. A higher generation rate therefore need not shorten the complete request by the same proportion. Several release-state changes also accompany the precision option, preventing an isolated FP8 causal claim.\n\n## The newer release is a separate experiment\n\nOctober 10 [Penny Royal 3.0 release metadata](https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/release-3.0.json) identifies another checkpoint, adaptive MTP and a 1,067,072-token pool. Its cold inputs are exactly 63,921 and 489,915 tokens; reported native mean prefill is 17,235.90 and 8,839.75 tokens/s, with TTFT 4.71 and 60.37 seconds. Three-key recall passes in the longer input. Completion length for those cold cases is not supplied here.\n\nThe 3.0 regular screens instead use 128 input and 1,024 output tokens, three repeats, temperature 0.8, top-p 0.95, top-k 40, min-p 0.05 and xhigh reasoning. Native means are 218.63 tokens/s at C1 and 489.35 aggregate at C4. They weight complete active reporting intervals, excluding idle/prefill-crossing intervals and partial tails. They do not measure complete agent-task wall time or replace the older long C4 cells. The accepted public core and the local checkout with private diagnostics have separate full identities. A document commit, public release image and actual native execution are separate provenance fields.\n\n## A four-card matrix provides a real campaign peak\n\nThe [CS Technology Services original report](https://www.cstech.dev/articles/qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell) uses four RTX PRO 6000 Max-Q cards, TP4, PCIe Gen4 without NVLink, vLLM `0.1.dev20073+g8e685d198` and harness v0.4.29. At nominal 128K, C1/C8/C16 aggregate rates are 128.5/505.0/729.9 tokens/s. Each cell lasts 30 seconds with `ignore_eos=true` and an 8,192-token output cap. Effective concurrency was checked. C32 is omitted for capacity, not an observed crash.\n\nIts 374,976 MiB sampled peak is summed used VRAM across the **whole matrix**, over 501 samples in 1,206 seconds. Per-cell peak and host peak RAM are absent. Matrix TTFT is warm-prefix/queue time; cold C1 takes 24.25 seconds at 128,021 actual input tokens. Per-user rates in that report are derived by division. Image digest, weight revision and full launch command were not retained. This is a different deployment, without a controlled comparison to Penny Royal.\n\n![Do not attach a peak from one campaign to an unrelated latency cell.](/assets/batch7-qwen-memory-scope.webp)\n\n*GiB and MiB are retained. Free, used, KV occupancy and host RAM are different quantities.*\n\nPenny Royal's 7.52 GiB free after graph capture is startup headroom. Its 1,897 MiB sampled minimum free belongs to ten GPU-media checks. Neither is the long C4 peak used memory. Keep free versus used, one versus four cards, startup versus workload and GPU versus host measurements separate. Pool occupancy is another counter again; percentages cannot be converted into whole-process bytes without an established mapping.\n\n## Sustained submissions can still queue\n\nThe [credited community follow-up](https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/COMMUNITY-RESULTS.md) describes a separate operator's v2.5.0 configuration with a 1,114,304-token pool. Ten rounds submit four approximately 240K requests with 8,192 reported output tokens each. The author reports 93–107 seconds per round, 40/40 HTTP 200 and no OOMs, retractions or aborts. Queuing is observed and pool occupancy peaks at 57 percent. This supports sustained operation for that workload; it does not establish four running throughout, near-full-pool stress, exact retrieval correctness, peak RAM or a changed runtime default. The approximate input label remains approximate.\n\n## A state-cache failure below the request ceiling\n\nAn [original SGLang incident comment](https://github.com/sgl-project/sglang/pull/37619#issuecomment-5606493113) records approximately 140K cold input, a 262K-class window, FP8 E4M3 KV, no speculation, 4,096-token prefill chunks, 51 Mamba slots and a 12-request ceiling. HiCache uses write-through. Only two requests are running when Mamba usage jumps from 0.18 to 1.00 while full-token usage is 0.46. The scheduler raises `Can not alloc mamba cache`; its child exits and the container reload interrupts service. The reporter mentions one queued request, while the cited failure log shows zero at that instant. Both observations are retained.\n\n![Original SGLang incident: approximately 140K cold prompt, HiCache write-through.](/assets/batch7-qwen-mamba-failure.webp)\n\n*Source-reported outage. The proposed fallback and CPU tests do not prove every GPU workload fixed.*\n\nThe related [PR37619](https://github.com/sgl-project/sglang/pull/37619) remains open and unmerged at this check. Its author reports two CPU tests and no rerun of the GPU reproducer. The proposed fallback skips an optional unfinished checkpoint when slots cannot be allocated. That scope does not establish a universal fix or a new speed result. Request slots, KV capacity and held Mamba states are separate constraints.\n\nFor deployment selection, first match hardware, exact checkpoint, runtime, cache policy and requested output. Then ask whether the source observes simultaneous execution and measures memory over the same interval. Preserve a null where that evidence is missing. This produces a more useful capacity decision than combining the fastest token rate, largest configured window and smallest memory number from different runs.\n" } ] }, { "id": "spark-deployment-score-audit", "title": "Read GLM, MiniMax and Kimi deployments with score limitations", "needs": [ "articles" ], "purpose": "Keep complete versions, raw medians, false quality gates, July/August dates, missing peak RAM, subset sizes and timeout retries.", "expression": "articles.articles.filter(a => ['A16','A17','A18'].includes(a.id))", "expected_output": [ { "id": "A16", "path": "/guides/glm53-flash-dual-spark-quality-gates/", "title": "GLM-5.3-Flash on Two Sparks: Speed Meets Quality Gates", "summary": "A pinned NVFP4 target, DFlash2 draft and community image reach sustained decode and 120124-token retrieval, while strict output failures remain visible.", "answer": "The selected seven-token draft arm is faster than the baseline in this source, but every arm retains quality_passed=false. The fastest three-token diagnostic failed the response contract before code execution. This is a text-only community experiment with useful negative results.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "Weschera", "repository": "Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "platform": "github", "revision": "281dc9a982bca17e944b322746658250a670ae73", "document_path": "README.md", "pinned_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/README.md", "document_sha256": "76327764bf18694718fbfac4d253c5525bf98e0b1d8d914f35765acfd34904cf", "last_commit_at": "2026-09-10T13:54:03Z", "url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "report_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/README.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "glm53-flash-nvfp4-dflash-quality" } ], "supplemental_sources": [ { "document_id": "glm-weschera-extra0", "parent_record_id": "glm53-flash-nvfp4-dflash-quality", "publisher": "Weschera", "repository": "Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "revision": "281dc9a982bca17e944b322746658250a670ae73", "document_path": "docs/RESULTS.md", "pinned_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/docs/RESULTS.md", "document_sha256": "ff3e50dd4f7a65232e37c01e4dd79ca5bee10c9ba66dbbec0f2cf1a7b5a4c3d5", "last_commit_at": "2026-09-10T13:54:03Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "glm-weschera-extra1", "parent_record_id": "glm53-flash-nvfp4-dflash-quality", "publisher": "Weschera", "repository": "Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "revision": "281dc9a982bca17e944b322746658250a670ae73", "document_path": "REPRODUCE.md", "pinned_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/REPRODUCE.md", "document_sha256": "89dd3e9334b41901f2ae8dc1d92ec8437e2ed90c143e4026c28c0978242f13c2", "last_commit_at": "2026-09-10T13:54:03Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "glm-weschera-extra2", "parent_record_id": "glm53-flash-nvfp4-dflash-quality", "publisher": "Weschera", "repository": "Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "revision": "281dc9a982bca17e944b322746658250a670ae73", "document_path": "SOURCE_MANIFEST.json", "pinned_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/SOURCE_MANIFEST.json", "document_sha256": "7065d7de93ef78feb6023048fefde4506c097148cfc2733bfe65257bcbac1692", "last_commit_at": "2026-09-10T13:54:03Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "glm-weschera-extra3", "parent_record_id": "glm53-flash-nvfp4-dflash-quality", "publisher": "Weschera", "repository": "Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "revision": "281dc9a982bca17e944b322746658250a670ae73", "document_path": "evidence/text128-dflash-resume/comparison.json", "pinned_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/evidence/text128-dflash-resume/comparison.json", "document_sha256": "36dc46ce255d484340558a64b6b45ce2b79f644471d91cbe6dc0e7a82de2d461", "last_commit_at": "2026-09-10T13:54:03Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "glm-weschera-extra5", "parent_record_id": "glm53-flash-nvfp4-dflash-quality", "publisher": "Weschera", "repository": "Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "revision": "281dc9a982bca17e944b322746658250a670ae73", "document_path": "evidence/text128-dflash-resume/dflash2-7/quality-report.json", "pinned_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/evidence/text128-dflash-resume/dflash2-7/quality-report.json", "document_sha256": "239adbba20e74bd853f04af15301d6c5b4254512b5ee9e4b79a532b64bac0957", "last_commit_at": "2026-09-10T13:54:03Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "glm-method-0", "parent_record_id": "glm53-flash-nvfp4-dflash-quality", "publisher": "Weschera", "repository": "Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark", "revision": "281dc9a982bca17e944b322746658250a670ae73", "document_path": "evidence/text128-dflash-resume/dflash2-7/speed-prose-0-request.json", "pinned_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/evidence/text128-dflash-resume/dflash2-7/speed-prose-0-request.json", "document_sha256": "46e917cee3ce542c4b710c92cdb0f22fa98281177d0405f94e57af3f6fd89ff6", "last_commit_at": null, "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" } ], "facts": [], "reported_results": [ { "label": "baseline median decode", "metric": "decode_rate", "value": 15.181243819412654, "unit": "tokens/s", "statistic": "median of four different prompts in one arm boot; not repeated identical task", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "nvidia/GLM-5.3-Flash-NVFP4", "model_revision": "423acf37583782c51c142d145aef733d72943d93", "runtime_execution_revision": null, "executed_image_digest": "sha256:4def0ef644cb2e9814136dcffd5e385e21bc594f48f3b292234051904abe85a6", "runtime_patch_source": "050081dc41ce6edd4d3f15fa19dc3410ba4210e3", "engine": "vLLM community GB10 image with Marlin W4A16 MoE", "system": "two 128 GB DGX Spark GB10, TP2 over RoCE; text only", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVIDIA NVFP4 storage; Marlin W4A16 MoE compute", "kv_dtype": "fp8_e4m3", "KV_allocation_GiB": 6, "context_limit_tokens": 131072, "configured_max_requests": 1, "concurrency": 1, "prefill_batch_tokens": 128, "enforce_eager": true, "cache_state": "prefix caching disabled", "thinking": "custom forced-OFF template; does not qualify as native OFF", "temperature": 0, "top_p": 1, "seed": 37, "repetitions": "four different sustained prompts in one boot per arm", "experiment_date": null, "metric_denominator": "completion tokens / (wall minus measured TTFT); estimated decode", "output_tokens": null, "actual_output_tokens": null, "peak_runtime_RAM": null, "draft_tokens": 0, "drafter": null, "drafter_revision": null, "input_tokens": [ 78, 92 ], "actual_completion_tokens_by_prompt": [ 4356, 2355, 10853, 2238 ], "all_speed_requests_natural_stop": true, "quality_passed": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-weschera-extra3", "source_document_sha256": "36dc46ce255d484340558a64b6b45ce2b79f644471d91cbe6dc0e7a82de2d461", "source_line": 18, "evidence_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/evidence/text128-dflash-resume/comparison.json#L18", "source_record_id": "glm53-flash-nvfp4-dflash-quality" }, { "label": "dflash2-7 median decode", "metric": "decode_rate", "value": 20.50006471628756, "unit": "tokens/s", "statistic": "median of four different prompts in one arm boot; not repeated identical task", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "nvidia/GLM-5.3-Flash-NVFP4", "model_revision": "423acf37583782c51c142d145aef733d72943d93", "runtime_execution_revision": null, "executed_image_digest": "sha256:4def0ef644cb2e9814136dcffd5e385e21bc594f48f3b292234051904abe85a6", "runtime_patch_source": "050081dc41ce6edd4d3f15fa19dc3410ba4210e3", "engine": "vLLM community GB10 image with Marlin W4A16 MoE", "system": "two 128 GB DGX Spark GB10, TP2 over RoCE; text only", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVIDIA NVFP4 storage; Marlin W4A16 MoE compute", "kv_dtype": "fp8_e4m3", "KV_allocation_GiB": 6, "context_limit_tokens": 131072, "configured_max_requests": 1, "concurrency": 1, "prefill_batch_tokens": 128, "enforce_eager": true, "cache_state": "prefix caching disabled", "thinking": "custom forced-OFF template; does not qualify as native OFF", "temperature": 0, "top_p": 1, "seed": 37, "repetitions": "four different sustained prompts in one boot per arm", "experiment_date": null, "metric_denominator": "completion tokens / (wall minus measured TTFT); estimated decode", "output_tokens": null, "actual_output_tokens": null, "peak_runtime_RAM": null, "draft_tokens": 7, "drafter": "incoai/GLM-5.3-Flash-DFlash2", "drafter_revision": "bf582e4eacc1810f76656d1811693ff6c6737d2a", "input_tokens": [ 78, 92 ], "actual_completion_tokens_by_prompt": [ 7396, 2272, 2292, 2158 ], "all_speed_requests_natural_stop": true, "quality_passed": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-weschera-extra3", "source_document_sha256": "36dc46ce255d484340558a64b6b45ce2b79f644471d91cbe6dc0e7a82de2d461", "source_line": 98, "evidence_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/evidence/text128-dflash-resume/comparison.json#L98", "source_record_id": "glm53-flash-nvfp4-dflash-quality" }, { "label": "dflash2-3 median decode", "metric": "decode_rate", "value": 24.77753188791761, "unit": "tokens/s", "statistic": "median of four different prompts in one arm boot; not repeated identical task", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "nvidia/GLM-5.3-Flash-NVFP4", "model_revision": "423acf37583782c51c142d145aef733d72943d93", "runtime_execution_revision": null, "executed_image_digest": "sha256:4def0ef644cb2e9814136dcffd5e385e21bc594f48f3b292234051904abe85a6", "runtime_patch_source": "050081dc41ce6edd4d3f15fa19dc3410ba4210e3", "engine": "vLLM community GB10 image with Marlin W4A16 MoE", "system": "two 128 GB DGX Spark GB10, TP2 over RoCE; text only", "node_count": 2, "parallelism": "TP2", "weight_precision": "NVIDIA NVFP4 storage; Marlin W4A16 MoE compute", "kv_dtype": "fp8_e4m3", "KV_allocation_GiB": 6, "context_limit_tokens": 131072, "configured_max_requests": 1, "concurrency": 1, "prefill_batch_tokens": 128, "enforce_eager": true, "cache_state": "prefix caching disabled", "thinking": "custom forced-OFF template; does not qualify as native OFF", "temperature": 0, "top_p": 1, "seed": 37, "repetitions": "four different sustained prompts in one boot per arm", "experiment_date": null, "metric_denominator": "completion tokens / (wall minus measured TTFT); estimated decode", "output_tokens": null, "actual_output_tokens": null, "peak_runtime_RAM": null, "draft_tokens": 3, "drafter": "incoai/GLM-5.3-Flash-DFlash2", "drafter_revision": "bf582e4eacc1810f76656d1811693ff6c6737d2a", "input_tokens": [ 78, 92 ], "actual_completion_tokens_by_prompt": [ 2822, 2194, 6331, 2105 ], "all_speed_requests_natural_stop": true, "quality_passed": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-weschera-extra3", "source_document_sha256": "36dc46ce255d484340558a64b6b45ce2b79f644471d91cbe6dc0e7a82de2d461", "source_line": 173, "evidence_url": "https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/evidence/text128-dflash-resume/comparison.json#L173", "source_record_id": "glm53-flash-nvfp4-dflash-quality" } ], "conditions": { "independent_site_runs": 0, "production_ready": false, "native_thinking_OFF_qualified": false }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch7 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch7-glm-hero.webp", "alt": "Two DGX Sparks / NVIDIA NVFP4 artifact / Marlin W4A16 compute / text only.", "width": 1600, "height": 900, "sha256": "b4c356d3f835bf5f2ebfcaab05a4ed9822fe62485d0ada02d57bd5ee618cc39c", "caption": "Four sustained prompts per arm, one boot per arm. No production-readiness score." }, "quality_comparisons": [ { "variant": "baseline", "speculative_tokens": 0, "quality_passed": false, "runtime_checks_passed": true, "semantic_checks_passed": false, "long_context_passed": true, "strict_failures": [ { "probe": "short-quality", "category": "semantic", "passed": false, "error": "Expected one code block" } ] }, { "variant": "dflash2-7", "speculative_tokens": 7, "quality_passed": false, "runtime_checks_passed": true, "semantic_checks_passed": true, "long_context_passed": true, "strict_failures": [ { "probe": "code", "requirement": "Only Python code, no Markdown fence", "passed": false }, { "probe": "thinking-on", "requirement": "Final content exactly 703", "passed": false, "observed": "**Reasoning:** The sum of the first n positive integers is given by the formula n(n+1)/2. For n = 37:\n\nSum = 37 × 38 / 2 = 37 × 19 = 703\n\n**Answer:** 703" } ] }, { "variant": "dflash2-3", "speculative_tokens": 3, "quality_passed": false, "runtime_checks_passed": true, "semantic_checks_passed": false, "long_context_passed": true, "strict_failures": [ { "probe": "short-quality", "category": "semantic", "passed": false, "error": "Expected one code block" } ] } ], "memory_observations": [ { "kind": "explicit_KV_allocation", "value": 6, "unit": "GiB", "whole_model_peak_RAM": null } ], "long_context_cases": [ { "classification": "source_reported_three_marker_success", "actual_input_tokens": 120124, "configured_limit_tokens": 131072, "arms": [ "baseline", "dflash2-7", "dflash2-3" ], "exact_all_three_markers": true, "full_window_tested": false, "general_long_context_reasoning_proven": false } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# GLM-5.3-Flash on Two Sparks: Speed Meets Quality Gates\n\n![Two DGX Sparks / NVIDIA NVFP4 artifact / Marlin W4A16 compute / text only.](/assets/batch7-glm-hero.webp)\n\n*Four sustained prompts per arm, one boot per arm. No production-readiness score.*\n\nA deployment can finish long requests and generate faster while still failing the contract an application needs. Weschera's [GLM-5.3-Flash experiment](https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/README.md) is useful because it preserves that outcome. On two DGX Sparks, a seven-token DFlash2 arm improves the source's median estimated decode rate, but strict output failures remain. A faster three-token diagnostic is rejected before code execution. None of the three arms receives a clean overall quality pass. These are original community observations, not new Run AI Here measurements.\n\n## Pin the target, draft and execution environment separately\n\nThe target is `nvidia/GLM-5.3-Flash-NVFP4` at `423acf37583782c51c142d145aef733d72943d93`. The DFlash2 draft is a different artifact at `bf582e4eacc1810f76656d1811693ff6c6737d2a`. The community vLLM image has a recorded digest ending `abe85a6`; its complete digest is retained in the adjacent JSON and original manifest. A document pin does not replace any of these identities.\n\nThe [source manifest](https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/SOURCE_MANIFEST.json) and [reproduction document](https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/REPRODUCE.md) describe two 128 GB unified-memory GB10 nodes using TP2 over RoCE. NVIDIA NVFP4 is the storage format, while Marlin uses W4A16 MoE compute. Do not describe the result as a native W4A4 throughput test. The target export lacks an MTP head; DFlash2 is the tested speculation path. The report is text only, with image and video inputs disabled.\n\nThe selected setup uses eager execution, an explicit 6 GiB FP8 E4M3 KV allocation, a 131,072-token window, one request slot, a 128-token prefill batch and disabled prefix caching. An overlay supplies Transformers 5.16.1 and tokenizers 0.23.1. These details make a useful reproduction inventory. They do not establish portability to a stock upstream installation or a second GB10 software stack, and this is a community recipe rather than an official NVIDIA production qualification.\n\n## Read the speed denominator before the headline\n\nThe [saved comparison JSON](https://github.com/Weschera/GLM-5.3-Flash-NVIDIA-NVFP4-2x-DGX-Spark/blob/281dc9a982bca17e944b322746658250a670ae73/evidence/text128-dflash-resume/comparison.json) includes four naturally stopping sustained requests per arm: two English technical tutorials and two Korean essays. Prompt counts are 78 or 92 tokens. Outputs vary greatly, rather than holding one identical response length constant. The saved English request uses temperature zero, top-p one and seed 37. Its output cap is the remaining serving window. The source does not accept a context-limit finish as a natural benchmark stop.\n\n| Arm | Draft tokens | Median estimated decode, tokens/s | Overall quality gate |\n|---|---:|---:|---|\n| Baseline | 0 | 15.18 | false |\n| DFlash2 selected | 7 | 20.50 | false |\n| DFlash2 diagnostic | 3 | 24.78 | false |\n\nEstimated decode divides completion tokens by wall time after subtracting measured TTFT. It is different from end-to-end request throughput. There is one boot per arm and four different prompts, not four independent boot replications of an identical task. No uncertainty interval is supplied. Ratios of those medians describe these outputs under these settings; they do not establish a universal improvement or the same reduction in application completion time.\n\n![Three GLM decode medians are shown alongside a false overall quality flag for every arm.](/assets/batch7-glm-speed-gates.webp)\n\n*The selected depth-7 arm passed executed semantic checks but retained two strict output-format failures.*\n\nFor example, the baseline English outputs contain 4,356 and 10,853 tokens, while the depth-7 English outputs contain 7,396 and 2,292. The amount and shape of generated work differ. Those facts matter when deciding whether a faster token stream improves a particular application. Short ACK latency, sustained prose generation, code correctness and long retrieval should remain separate checks.\n\n## Preserve the selected arm's strict failures\n\nThe selected depth-7 arm passes the executed semantic checks, including the source's fixed and generated code cases. It nevertheless fails two strict requirements. One probe asks for Python without a Markdown fence and receives fenced code. A separate thinking-on probe requires final content exactly `703` and receives reasoning plus the answer. These are observable response-contract failures. The arithmetic answer being present does not turn the strict final-content requirement into a pass.\n\nAll arms have `quality_passed=false` in the raw comparison. A selected serving candidate is therefore not synonymous with a fully qualified deployment. The distinction matters for JSON consumers, code runners and tools that accept a constrained output format. An editor should not strip unwanted material, run the repaired answer and retrospectively mark the original response compliant. A downstream repair workflow would need its own explicitly defined evaluation.\n\n## Why the fastest arm was rejected\n\nThe baseline and depth-3 diagnostic fail the early one-code-block contract in the short quality gate. The source stops before later code execution checks in that branch. This means later checks are **skipped**, not that the underlying algorithm was measured and proven wrong. The failed response structure is enough to reject that arm for the source's contract, but it does not justify a broader claim about every code answer from the model.\n\nLikewise, the comparison's `semantic_checks_passed=false` must be read with the detailed gate. It is a campaign outcome containing an early contract failure. A single Boolean cannot explain which tests executed. The adjacent JSON retains both the flag and strict-failure descriptions, so people and AI can inspect the actual reason rather than infer it from a speed ranking.\n\n![120,124 actual input tokens under a 131,072-token configured window.](/assets/batch7-glm-context.webp)\n\n*All three arms recovered three markers; this is neither full-window proof nor broad quality equivalence.*\n\n## The long input is an exact retrieval check\n\nAll three arms recover beginning, middle and end markers from 120,124 actual input tokens. The configured window is 131,072. The check demonstrates that particular retrieval under that serving configuration; it does not prove a completed full-window input, arbitrary document reasoning or concurrent long-context admission. These requests run at C1, and no higher-concurrency performance sweep is supplied.\n\nThe 6 GiB KV allocation is a configuration value. It is not whole-model peak unified memory. The source does not provide a paired total peak-RAM trace for every sustained or long-context request. Weight-file bytes are another quantity and should not be added to an allocation to invent an observed peak. Peak memory, repeated-boot dispersion and concurrency scaling stay unknown in this interpretation.\n\n## Treat the custom thinking template as its own condition\n\nThe report's forced-OFF template uses an empty thinking prefix while retaining maximum-effort system wording. Planning can leak into final text. The author therefore does not qualify it as native thinking OFF. The separate thinking-on strict failure should not be attributed to that template without an isolated comparison. Output structure, reasoning controls and draft depth vary across the probes, and the saved failures do not identify one universal cause.\n\n![A pinned NVIDIA artifact does not make a community GB10 recipe an official NVIDIA runtime.](/assets/batch7-glm-identity.webp)\n\n*DFlash2 draft revision, target revision and image digest remain separate source fields.*\n\nFinally, the selected server's restart is followed by a health probe. That is a service-readiness observation, not a repeated quality suite on the final boot. Anyone reusing this recipe should retain the target revision, draft revision, image digest, template and overlay in their own inventory, then validate the response contracts they actually need. The published experiment gives concrete starting conditions and useful failures; it leaves production readiness, broader quality and workload-specific peak memory to further measured evidence.\n" }, { "id": "A17", "path": "/guides/minimax-m3-dual-spark-measured-lane/", "title": "MiniMax-M3 on Two Sparks: The Measured W4A16 Lane", "summary": "Separate a July lane-A reproduction from proposed speed/context recipes, retaining request timing, startup memory brackets and a negative NCCL result.", "answer": "Only lane A is reproduced by this reporter: W4A16 GPTQ with EAGLE3 and NVFP4 KV on two Sparks. Its four short workload observations use whole request wall time. Lanes B and C remain another author’s references under test, and the source supplies no quality score or measured peak RAM.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "tonyd2wild", "repository": "tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s", "platform": "github", "revision": "a8101fa423cac8140016e45110a2ccc8a0a72162", "document_path": "README.md", "pinned_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/README.md", "document_sha256": "98f2ec691a993fb8f143ca6f31d19ad9c2090de979e577b8a4dfbc4c8039a6b2", "last_commit_at": "2026-07-13T01:19:48Z", "url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s", "report_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/README.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "minimax-m3-tp2-lane-a" } ], "supplemental_sources": [ { "document_id": "minimax-tp2-extra0", "parent_record_id": "minimax-m3-tp2-lane-a", "publisher": "tonyd2wild", "repository": "tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s", "revision": "a8101fa423cac8140016e45110a2ccc8a0a72162", "document_path": "DEFAULT-CONFIG.md", "pinned_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/DEFAULT-CONFIG.md", "document_sha256": "04d26375babfc741c02dac6d1d8c0f0004152284f4843c2ca1efe997d2282287", "last_commit_at": "2026-07-13T01:19:48Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "minimax-tp2-extra1", "parent_record_id": "minimax-m3-tp2-lane-a", "publisher": "tonyd2wild", "repository": "tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s", "revision": "a8101fa423cac8140016e45110a2ccc8a0a72162", "document_path": "benchmarks/RESULTS.md", "pinned_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/benchmarks/RESULTS.md", "document_sha256": "9f3d73849567604e7dc04c19900c1e26343fb1193efb7b5df35fa2ade9838cad", "last_commit_at": "2026-07-13T01:19:48Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" } ], "facts": [], "reported_results": [ { "label": "JSON thinking False", "metric": "request_output_rate", "value": 36.6, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "MiniMax-M3", "model_artifact": "Sebesky/MiniMax-M3-W4A16-GPTQ", "model_revision": null, "runtime_execution_revision": null, "runtime_source_prefix": "979b56a66c96", "executed_image_digest": null, "image_tag": "vllm-node-minimax-m3-b12x", "engine": "vLLM + b12x", "system": "two DGX Spark GB10, TP2 over direct 200G RoCE", "node_count": 2, "parallelism": "TP2", "weight_precision": "W4A16 GPTQ", "kv_dtype": "b12x NVFP4", "drafter": "Sebesky/MiniMax-M3-EAGLE3-RTN-INT4", "drafter_revision": null, "draft_tokens": 3, "context_limit_tokens": 196608, "KV_pool_tokens": 208128, "configured_max_requests": 1, "concurrency": 1, "input_tokens": null, "output_tokens": 256, "actual_output_tokens": null, "temperature": 0, "top_p": null, "cache_state": "warm engine; first JIT request excluded", "repetitions": null, "experiment_date": "2026-07-05", "GMU": 0.929, "quality_score": null, "peak_runtime_RAM": null, "metric_denominator": "completion_tokens / whole request wall including prefill", "workload": "JSON", "thinking": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "minimax-tp2-extra1", "source_document_sha256": "9f3d73849567604e7dc04c19900c1e26343fb1193efb7b5df35fa2ade9838cad", "source_line": 38, "evidence_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/benchmarks/RESULTS.md#L38", "source_record_id": "minimax-m3-tp2-lane-a" }, { "label": "JSON thinking adaptive", "metric": "request_output_rate", "value": 35.4, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "MiniMax-M3", "model_artifact": "Sebesky/MiniMax-M3-W4A16-GPTQ", "model_revision": null, "runtime_execution_revision": null, "runtime_source_prefix": "979b56a66c96", "executed_image_digest": null, "image_tag": "vllm-node-minimax-m3-b12x", "engine": "vLLM + b12x", "system": "two DGX Spark GB10, TP2 over direct 200G RoCE", "node_count": 2, "parallelism": "TP2", "weight_precision": "W4A16 GPTQ", "kv_dtype": "b12x NVFP4", "drafter": "Sebesky/MiniMax-M3-EAGLE3-RTN-INT4", "drafter_revision": null, "draft_tokens": 3, "context_limit_tokens": 196608, "KV_pool_tokens": 208128, "configured_max_requests": 1, "concurrency": 1, "input_tokens": null, "output_tokens": 256, "actual_output_tokens": null, "temperature": 0, "top_p": null, "cache_state": "warm engine; first JIT request excluded", "repetitions": null, "experiment_date": "2026-07-05", "GMU": 0.929, "quality_score": null, "peak_runtime_RAM": null, "metric_denominator": "completion_tokens / whole request wall including prefill", "workload": "JSON", "thinking": "adaptive" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "minimax-tp2-extra1", "source_document_sha256": "9f3d73849567604e7dc04c19900c1e26343fb1193efb7b5df35fa2ade9838cad", "source_line": 38, "evidence_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/benchmarks/RESULTS.md#L38", "source_record_id": "minimax-m3-tp2-lane-a" }, { "label": "code thinking False", "metric": "request_output_rate", "value": 31.8, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "MiniMax-M3", "model_artifact": "Sebesky/MiniMax-M3-W4A16-GPTQ", "model_revision": null, "runtime_execution_revision": null, "runtime_source_prefix": "979b56a66c96", "executed_image_digest": null, "image_tag": "vllm-node-minimax-m3-b12x", "engine": "vLLM + b12x", "system": "two DGX Spark GB10, TP2 over direct 200G RoCE", "node_count": 2, "parallelism": "TP2", "weight_precision": "W4A16 GPTQ", "kv_dtype": "b12x NVFP4", "drafter": "Sebesky/MiniMax-M3-EAGLE3-RTN-INT4", "drafter_revision": null, "draft_tokens": 3, "context_limit_tokens": 196608, "KV_pool_tokens": 208128, "configured_max_requests": 1, "concurrency": 1, "input_tokens": null, "output_tokens": 256, "actual_output_tokens": null, "temperature": 0, "top_p": null, "cache_state": "warm engine; first JIT request excluded", "repetitions": null, "experiment_date": "2026-07-05", "GMU": 0.929, "quality_score": null, "peak_runtime_RAM": null, "metric_denominator": "completion_tokens / whole request wall including prefill", "workload": "code", "thinking": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "minimax-tp2-extra1", "source_document_sha256": "9f3d73849567604e7dc04c19900c1e26343fb1193efb7b5df35fa2ade9838cad", "source_line": 39, "evidence_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/benchmarks/RESULTS.md#L39", "source_record_id": "minimax-m3-tp2-lane-a" }, { "label": "code thinking adaptive", "metric": "request_output_rate", "value": 29.2, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "MiniMax-M3", "model_artifact": "Sebesky/MiniMax-M3-W4A16-GPTQ", "model_revision": null, "runtime_execution_revision": null, "runtime_source_prefix": "979b56a66c96", "executed_image_digest": null, "image_tag": "vllm-node-minimax-m3-b12x", "engine": "vLLM + b12x", "system": "two DGX Spark GB10, TP2 over direct 200G RoCE", "node_count": 2, "parallelism": "TP2", "weight_precision": "W4A16 GPTQ", "kv_dtype": "b12x NVFP4", "drafter": "Sebesky/MiniMax-M3-EAGLE3-RTN-INT4", "drafter_revision": null, "draft_tokens": 3, "context_limit_tokens": 196608, "KV_pool_tokens": 208128, "configured_max_requests": 1, "concurrency": 1, "input_tokens": null, "output_tokens": 256, "actual_output_tokens": null, "temperature": 0, "top_p": null, "cache_state": "warm engine; first JIT request excluded", "repetitions": null, "experiment_date": "2026-07-05", "GMU": 0.929, "quality_score": null, "peak_runtime_RAM": null, "metric_denominator": "completion_tokens / whole request wall including prefill", "workload": "code", "thinking": "adaptive" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "minimax-tp2-extra1", "source_document_sha256": "9f3d73849567604e7dc04c19900c1e26343fb1193efb7b5df35fa2ade9838cad", "source_line": 39, "evidence_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/benchmarks/RESULTS.md#L39", "source_record_id": "minimax-m3-tp2-lane-a" }, { "label": "Lane A KV pool", "metric": "KV_pool", "value": 208128, "unit": "tokens", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "MiniMax-M3", "model_artifact": "Sebesky/MiniMax-M3-W4A16-GPTQ", "model_revision": null, "runtime_execution_revision": null, "runtime_source_prefix": "979b56a66c96", "executed_image_digest": null, "image_tag": "vllm-node-minimax-m3-b12x", "engine": "vLLM + b12x", "system": "two DGX Spark GB10, TP2 over direct 200G RoCE", "node_count": 2, "parallelism": "TP2", "weight_precision": "W4A16 GPTQ", "kv_dtype": "b12x NVFP4", "drafter": "Sebesky/MiniMax-M3-EAGLE3-RTN-INT4", "drafter_revision": null, "draft_tokens": 3, "context_limit_tokens": 196608, "KV_pool_tokens": 208128, "configured_max_requests": 1, "concurrency": 1, "input_tokens": null, "output_tokens": 256, "actual_output_tokens": null, "temperature": 0, "top_p": null, "cache_state": "warm engine; first JIT request excluded", "repetitions": null, "experiment_date": "2026-07-05", "GMU": 0.929, "quality_score": null, "peak_runtime_RAM": null, "metric_denominator": "completion_tokens / whole request wall including prefill", "thinking": null }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "minimax-tp2-extra1", "source_document_sha256": "9f3d73849567604e7dc04c19900c1e26343fb1193efb7b5df35fa2ade9838cad", "source_line": 31, "evidence_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/benchmarks/RESULTS.md#L31", "source_record_id": "minimax-m3-tp2-lane-a" } ], "conditions": { "independent_site_runs": 0, "measurement_date": "2026-07-05", "only_lane_A_reproduced_by_this_reporter": true, "concurrency_measured_above_one": false }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch7 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch7-minimax-hero.webp", "alt": "July 5, 2026 measurements / W4A16 GPTQ / EAGLE3 depth 3 / b12x NVFP4 KV.", "width": 1600, "height": 900, "sha256": "e485ab10218d38fa58c47dc2aa854b93c8457761da51dc897ac7ccf9ab834216", "caption": "The model family is current; these particular tests retain their July date." }, "case_reports": [ { "id": "minimax-lane-a-startup-bracket", "classification": "source_reported_startup_capacity_failure", "GMU_failed": 0.93, "shortfall_GiB": 0.02, "GMU_boots_but_KV_below_requested": 0.925, "GMU_selected": 0.929, "startup_margin_source_label": "approximately 100 MB", "runtime_peak_RAM": null, "source_url": "https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/DEFAULT-CONFIG.md" } ], "memory_observations": [ { "kind": "KV_pool_tokens_not_peak_RAM", "value": 208128, "unit": "tokens", "configured_limit_tokens": 196608, "whole_model_peak_RAM": null } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# MiniMax-M3 on Two Sparks: The Measured W4A16 Lane\n\n![July 5, 2026 measurements / W4A16 GPTQ / EAGLE3 depth 3 / b12x NVFP4 KV.](/assets/batch7-minimax-hero.webp)\n\n*The model family is current; these particular tests retain their July date.*\n\nMiniMax-M3 has several community deployment recipes, but their evidence should not be collapsed into a single “best” configuration. The [tonyd2wild report](https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/README.md) distinguishes a lane independently reproduced on its own two Sparks from two alternative lanes still under test. This note retains that distinction, the original July measurement date, short-probe timing and a negative transport experiment. It offers a concrete deployment reference without turning unfinished recipes into completed benchmarks.\n\n## Keep the measurement date attached to the model\n\nLane A is verified on July 5, 2026. Reading it today as a MiniMax-family deployment reference does not make its results October measurements. Its hardware is two DGX Sparks with 128 GB unified memory each, connected by direct 200G RoCE and serving through TP2. This is distinct from a three-Spark layer split or an offload configuration on a discrete GPU workstation.\n\nThe target is `Sebesky/MiniMax-M3-W4A16-GPTQ`, with a separate `Sebesky/MiniMax-M3-EAGLE3-RTN-INT4` draft. The report describes the full, unpruned model and a vLLM+b12x path. It names an execution image tag and a short source reference, but does not retain a full executed image digest or full target/draft weight revisions. Those fields remain null. Looking up a current model repository today would not establish which exact files ran in July.\n\nThe source credits a3refaat's upstream recipe and clearly labels where its own reproduction begins. Attribution is important here: a reference number quoted from the upstream author is not an additional independent measurement by the second reporter. The catalog points below are restricted to the reporter's measured lane A.\n\n## The measured lane uses one stream\n\nThe [default-configuration document](https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/DEFAULT-CONFIG.md) selects EAGLE3 depth three, b12x NVFP4 KV, a 196,608-token configured context and a 208,128-token KV pool. All published lane-A speed observations use `max_num_seqs=1`. A configured large context does not mean those short probes contain that many input tokens or test the whole window.\n\nThe [original result document](https://github.com/tonyd2wild/MiniMax-M3-2x-DGX-Spark-36-tok-s/blob/a8101fa423cac8140016e45110a2ccc8a0a72162/benchmarks/RESULTS.md) specifies 256-token probes, temperature zero, one stream and a warm engine. The first request after boot is excluded because b12x compilation warmup can take about a minute. Actual per-probe input counts, completion counts and repeated-run dispersion are not supplied. Although the source table says “decode,” its stated denominator is `completion_tokens / wall`, including request wall time. The article and JSON keep that definition.\n\n| Measured workload | Thinking disabled, tokens/s | Adaptive, tokens/s |\n|---|---:|---:|\n| JSON | 36.6 | 35.4 |\n| Code | 31.8 | 29.2 |\n\nThese four observations are useful for their own prompt classes. They are not a concurrency curve, a full-context curve or a confidence interval. Thinking disabled and adaptive can also produce different work, so the rate difference alone does not establish the same change in complete task time. No general quality score accompanies this experiment.\n\n![MiniMax-M3 request rates for JSON and code with thinking off and adaptive modes.](/assets/batch7-minimax-rates.webp)\n\n*The denominator is whole request wall time, despite the source table calling the values decode speed.*\n\nFor a latency-sensitive application, the next relevant evidence would record actual input and completion lengths, first-token latency, the complete request time and successful output checks under the intended mode. The existing table cannot supply those missing measurements. Keeping them explicit is more useful than filling them from another MiniMax report on a different engine or date.\n\n## The allocation bracket explains a reproducibility trap\n\nThe reporter finds that GMU 0.93 fails a startup free-memory check by 0.02 GiB. Reducing to 0.925 starts the engine but leaves a KV pool below the desired context requirement. The chosen 0.929 reaches the target pool with approximately 100 MB startup margin. The source preserves the bracket because one seemingly reasonable rounded parameter can change whether the recipe starts and whether it has enough configured capacity.\n\nThat margin is measured at startup in the reporter's boot state. It is not a sampled worst-case workload margin and does not show safe headroom for simultaneous long prefills. File-cache state, other resident processes and temporary runtime allocations can differ across systems. The published bracket does not justify a universal GMU recommendation or a guarantee that the same fraction works on every Spark pair.\n\n![GMU brackets are observations from the reporter's own boot state.](/assets/batch7-minimax-capacity.webp)\n\n*The approximately 100 MB startup margin is not a workload safety allowance.*\n\nThe 208,128-token pool is measured/configured KV capacity, in tokens. The 196,608-token ceiling is a per-request setting. They are not memory in bytes, and neither value is whole-model peak unified RAM. The source does not provide a paired peak-memory time series for a long request or multi-user load. Those unknowns remain null rather than being estimated from weight-file size or a nominal machine total.\n\n## A transport change that made this workload slower\n\nThe report tests `NCCL_PROTO=LL` with `NCCL_MAX_NCHANNELS=2`. In the code workload, stock NCCL gives 31.8 tokens/s and the LL/two-channel setting gives 28.8. The JSON workload also declines in the source's observed range. This negative result is retained rather than selecting only the fastest headline.\n\nThe author suggests that a larger MoE's communication pattern may explain why a setting useful on another model loses here. That is a hypothesis, not an isolated mechanism test. The evidence supports a narrower conclusion: this exact change did not improve the tested lane-A workloads on this pair. It does not identify a universally best NCCL protocol, prove a model-class law or replace measurements on another interconnect.\n\n![Lane A, stock NCCL versus LL protocol plus two channels.](/assets/batch7-minimax-negative.webp)\n\n*Source hypothesis about model class is interpretation, not a separately isolated mechanism.*\n\n## Why the other two lanes stay references\n\nLane C proposes a different KVarN KV path plus EAGLE3 for speed, and lane B proposes a drafterless context-oriented path. Their numbers in this repository are attributed to the upstream author's own hardware. The second reporter labels both lanes under test and leaves its own result checkboxes incomplete. This article does not publish their quoted speeds as reproduced lane-A data or add them to the selected result points.\n\nThe same restriction applies to proposed concurrency and repetition-penalty experiments. A checklist item is a plan. It cannot be turned into a success case because the surrounding recipe is detailed. Any later measured lane should retain its own boot evidence, KV pool, exact runtime identity, prompts, output limits, timing definitions and validation outcomes before joining this series.\n\nFor deployment selection, lane A is the supported starting reference in this source. Use its original reproduction inventory to understand the required kernel path, two-node transport and draft model, then account for missing identities and workload telemetry. The useful result is a bounded, reproducible question: whether the same lane satisfies a specified application's response and memory requirements. The published four short rates alone cannot answer the broader questions of maximum concurrency, long-context reliability, peak memory or standardized model quality.\n" }, { "id": "A18", "path": "/guides/kimi-k3-iq1s-spark-scores-retries/", "title": "Kimi IQ1_S on Three Sparks: Scores, Retries and Missing Pins", "summary": "Audit a publisher-labelled K3 layer-split deployment, approximate memory and small GSM8K/HumanEval subsets, retaining first-pass failures and timeout retries.", "answer": "The source reports 49/50 on a GSM8K subset and a HumanEval composite of 18/20 after retrying four timed-out problems. The initial HumanEval result was 14/20. Unequal cloud sampling, unpinned weight identity and small subsets do not prove lossless IQ1_S quantization or full 1M-input capability.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "platform": "github", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "README.md", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md", "document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "last_commit_at": "2026-08-07T15:45:04Z", "url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark", "report_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "kimi-k3-iq1s-three-spark-scores" } ], "supplemental_sources": [ { "document_id": "kimi-iq1s-extra0", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/canon-gsm8k-k3-local.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-local.json", "document_sha256": "2d297e3f59c7beb393fa72e6297edec8af0b6525139057ee0e26b425d071f2be", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra1", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/humaneval-p18081.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-p18081.json", "document_sha256": "29a942e4a2194c737b8492b59ab752015896fc8e3180cc85135c33c6b249f590", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra2", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/humaneval-retry-p18081.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-retry-p18081.json", "document_sha256": "469ff6a70092ca219c70e3dc2d6967729133f74e97ed0b9cd56e68323b2f2053", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra3", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/cloud-humaneval-logic-k3-cloud.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/cloud-humaneval-logic-k3-cloud.json", "document_sha256": "8d42fd9033e794d5a8a7330d5c56707128b521f869c46be071a364c5f45fa22c", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra4", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/canon-gsm8k-k3-cloud.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-cloud.json", "document_sha256": "636e0ee2c31986d961ffe637927bcb95aa1d3d82d78dea08f7efe0807f29971b", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" } ], "facts": [], "reported_results": [ { "label": "steady decode", "metric": "decode_rate", "value": 2.9, "unit": "tokens/s", "statistic": "approximate source summary", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "steady decode", "input_length_source_label": null }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 43, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L43", "source_record_id": "kimi-k3-iq1s-three-spark-scores" }, { "label": "8K prefill", "metric": "prefill_rate", "value": 79.6, "unit": "tokens/s", "statistic": "source summary; exact per-run token counts not supplied", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "8K prefill", "input_length_source_label": "8K" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 44, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L44", "source_record_id": "kimi-k3-iq1s-three-spark-scores" }, { "label": "32K prefill", "metric": "prefill_rate", "value": 43.9, "unit": "tokens/s", "statistic": "source summary; exact per-run token counts not supplied", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "32K prefill", "input_length_source_label": "32K" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 45, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L45", "source_record_id": "kimi-k3-iq1s-three-spark-scores" }, { "label": "131K prefill", "metric": "prefill_rate", "value": 48.5, "unit": "tokens/s", "statistic": "source summary; exact per-run token counts not supplied", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "131K prefill", "input_length_source_label": "131K" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 46, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L46", "source_record_id": "kimi-k3-iq1s-three-spark-scores" }, { "label": "GSM8K local subset", "metric": "accuracy", "value": 98, "unit": "percent", "statistic": "source fixed-subset score, no confidence interval", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "GSM8K canonical test subset", "correct": 49, "total": 50, "subset_ids": "50 source-indexed questions; fixed subset, not full benchmark", "timeout_seconds": 1800, "retry_policy": "GLM comparison changed output cap; local retries unreported" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-extra0", "source_document_sha256": "2d297e3f59c7beb393fa72e6297edec8af0b6525139057ee0e26b425d071f2be", "source_line": 5, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-local.json#L5", "source_record_id": "kimi-k3-iq1s-three-spark-scores" }, { "label": "HumanEval initial local subset", "metric": "accuracy", "value": 70, "unit": "percent", "statistic": "source fixed-subset score, no confidence interval", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval subset IDs 0 through 19", "correct": 14, "total": 20, "subset_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], "timeout_seconds": 600, "timeouts": 4, "retry_policy": "initial run; four timeouts retained as failures" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-extra1", "source_document_sha256": "29a942e4a2194c737b8492b59ab752015896fc8e3180cc85135c33c6b249f590", "source_line": 6, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-p18081.json#L6", "source_record_id": "kimi-k3-iq1s-three-spark-scores" }, { "label": "HumanEval timeout retries", "metric": "accuracy", "value": 100, "unit": "percent", "statistic": "source fixed-subset score, no confidence interval", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval timeout-only subset", "correct": 4, "total": 4, "subset_ids": [ 1, 6, 10, 17 ], "timeout_seconds": null, "retry_policy": "rerun only four timed-out problems with higher limits; not fresh 20-problem pass" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-extra2", "source_document_sha256": "469ff6a70092ca219c70e3dc2d6967729133f74e97ed0b9cd56e68323b2f2053", "source_line": 6, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-retry-p18081.json#L6", "source_record_id": "kimi-k3-iq1s-three-spark-scores" }, { "label": "HumanEval local after selected retries", "metric": "accuracy", "value": 90, "unit": "percent", "statistic": "source composite score after retries", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval subset IDs 0 through 19", "correct": 18, "total": 20, "subset_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], "timeout_seconds": null, "retry_policy": "14 initial passes plus 4 successful timeout-only retries; not first-pass 18/20", "failed_ids": [ 2, 7 ] }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 83, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L83", "source_record_id": "kimi-k3-iq1s-three-spark-scores" } ], "conditions": { "independent_site_runs": 0, "full_1048576_input_test_proven": false, "unquantized_local_control": false, "sampling_matched_to_cloud": false, "quantization_equivalence_proven": false, "upstream_checkpoint_identity_verified": false }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch7 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch7-kimi-hero.webp", "alt": "Publisher-labelled K3 / three Spark layer-split RPC / exact upstream checkpoint unpinned.", "width": 1600, "height": 900, "sha256": "a58430f51f1cabd9cec6b65a4f80966cbfed0bd0f9b42e79d3414132558834e0", "caption": "Small source subsets and unequal cloud settings do not establish lossless quantization." }, "quality_comparisons": [ { "benchmark": "GSM8K", "subset_size": 50, "local_correct": 49, "cloud_Kimi_correct": 49, "cloud_GLM52_correct": 49, "local_temperature": 0.2, "cloud_Kimi_temperature": 1, "local_output_cap": 8192, "cloud_Kimi_output_cap": 2048, "GLM_cap_changed_after_exhaustion": true, "matched_accuracy_proves_equivalence": false }, { "benchmark": "HumanEval", "subset_size": 20, "initial_local_correct": 14, "initial_timeouts": 4, "retry_subset_ids": [ 1, 6, 10, 17 ], "retry_correct": 4, "final_local_correct": 18, "remaining_failure_ids": [ 2, 7 ], "first_pass_18_of_20": false } ], "memory_observations": [ { "kind": "approximate_per_node_occupancy_unknown_measurement_method", "source_range_GB": [ 119, 120 ], "unit": "GB (source label; byte convention unspecified)", "node_count": 3, "is_measured_peak": false, "peak_RAM": null } ], "long_context_cases": [ { "classification": "configured_context_not_completed_full_window", "configured_limit_tokens": 1048576, "longest_performance_source_label": "131K", "actual_full_window_prompt_tokens": null } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# Kimi IQ1_S on Three Sparks: Scores, Retries and Missing Pins\n\n![Publisher-labelled K3 / three Spark layer-split RPC / exact upstream checkpoint unpinned.](/assets/batch7-kimi-hero.webp)\n\n*Small source subsets and unequal cloud settings do not establish lossless quantization.*\n\nSmall local quality reports can be informative when they preserve the questions tested, failure history and sampling controls. They become misleading when a composite score is presented as a first-pass result or matching percentages become a claim of lossless quantization. The [chishiki37 report](https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md) provides deployment details and raw subset results for a publisher-labelled Kimi K3 IQ1_S on three Sparks. This note keeps its useful observations and the limits visible together.\n\n## Start with the identity that is actually known\n\nThe August 7 report calls the model Kimi K3/Kimi-Linear IQ1_S, with nine GGUF shards and an approximately 330 GB file total. It names a shard pattern but does not pin the exact upstream repository revision, weight digest or complete runtime commit. Run AI Here therefore retains the publisher's model label without certifying that it uniquely identifies a current official checkpoint. File size and a plausible filename cannot establish that identity.\n\nThree DGX Sparks supply 128 GB unified memory each. A pwilkin llama.cpp fork uses layer-split RPC with two remote workers. This is not tensor parallelism across three GPUs. The fabric overview says MTU9000, while the actual deployment configuration says MTU4096, inline316 and receive-depth24. Those are distinct source statements, and the JSON preserves both instead of quietly choosing the larger value.\n\nThe configuration uses f16 KV, load mode `none` and one request slot. The source reports that Q8 KV is incompatible with its V-head dimension and block constraint in this build. That is a specific runtime/artifact condition, not a claim that every Kimi model or future llama.cpp version lacks quantized KV support. The original launch line is a reproduction reference; this site has not executed it.\n\n## A configured million-token window is not a measured full input\n\nThe configured window is 1,048,576 tokens. The performance summary reports approximately 2.9 tokens/s steady decode and the following prefill values. Exact per-run prompt counts, output lengths, timing traces and repeated-run dispersion are not supplied in that table.\n\n| Original input label | Reported prefill, tokens/s |\n|---|---:|\n| 8K | 79.6 |\n| 32K | 43.9 |\n| 131K | 48.5 |\n\n![Kimi IQ1_S source prefill rates at 8K, 32K and 131K nominal prompt lengths.](/assets/batch7-kimi-prefill.webp)\n\n*A configured 1,048,576-token window is separate from these performance observations.*\n\nThe longest listed performance label is 131K. A source statement that a server loads with a 1M window is not enough to establish a completed 1,048,576-token input, correct recall throughout that input or acceptable concurrency at that length. The selected sources do not supply the corresponding full-input payload and paired telemetry. Those fields stay unknown rather than borrowing a test from a different Kimi configuration.\n\n## Read the GSM8K score as a fixed subset\n\nThe [local GSM8K raw summary](https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-local.json) reports 49 correct of 50, or 98 percent, on the selected canonical test questions. The source's cloud Kimi and cloud GLM-5.2 comparisons also report 49/50 and the same failed item. These are source subsets, not full standardized leaderboard evaluations and not independent Run AI Here measurements.\n\nThe controls are not matched across services. Local Kimi uses temperature0.2 and an 8,192-token output cap, while cloud Kimi forces temperature1 and uses a 2,048-token GSM8K cap. The GLM comparison raises its cap after two token-exhaustion failures. Exact API model versions and a matched unquantized local baseline are missing. Equal subset percentages under those conditions do not establish equivalence or isolate the quality cost of IQ1_S quantization.\n\n## HumanEval: the first pass was 14/20\n\nThe [initial local HumanEval file](https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-p18081.json) records 14 passes out of 20, or 70 percent. Four requests time out at the 600-second client limit: IDs1,6,10 and17. Two other tasks, IDs2 and7, fail assertions. Timeouts are retained as first-pass failures rather than silently removed from the denominator.\n\nThe [timeout-only rerun](https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-retry-p18081.json) evaluates those four timed-out tasks with higher limits and passes all four. Its 4/4 result is a selected retry subset. Combined with the fourteen initial passes, it explains the README's final 18/20, or 90 percent. That composite is not a fresh first-pass evaluation of twenty problems and should never replace the initial 14/20 in a reliability discussion.\n\n![Kimi local HumanEval subset rises from 14 of 20 initially to 18 of 20 after four timeout retries.](/assets/batch7-kimi-retry-scores.webp)\n\n*The four successful retries are a selected subset; final 18/20 must retain this history.*\n\nOne retry takes 995.8 seconds and another 1,180.1. This matters operationally: raising a timeout can recover a correct answer while increasing how long a client or worker remains occupied. The source's final aggregate timing summary does not give a clean first-pass service-latency distribution. It is unsuitable for an application percentile claim, and the selected retries do not estimate the rate at which arbitrary future timeout failures recover.\n\nThe remaining failures concern `truncate_number` and `filter_by_substring`. The README groups both under floating-point edge cases, but substring filtering is not itself a floating-point task. This article retains the assertion failures without adopting that blanket diagnosis. Raw failure IDs and errors are evidence; a common root cause would require further analysis. No upstream code or generated candidate was executed during this editorial review.\n\n## Memory occupancy is approximate and lacks a peak method\n\nThe source gives about 119–120 GB per node and about 357 GB total, with small reported headroom. It does not define a peak sampling method, sampling cadence, process-versus-host boundary or paired context length. The GB label also lacks a verified byte convention. Those observations are approximate occupancy reports, not a measured peak-RAM curve.\n\n![The report labels approximately 119–120 GB per node without a telemetry method.](/assets/batch7-kimi-memory.webp)\n\n*Do not infer a 1M full-input pass, safe peak headroom or quantization equivalence.*\n\nThe reported `vmstat` swap-in/out counters are zero during observation. That does not prove no swap was allocated, that no future workload can swap or that all three machines have safe transient headroom at the configured maximum. A warm-cache load time and a memory occupancy snapshot answer different questions. They should not be added to weight-file bytes or transformed into a universal hardware sizing guarantee.\n\n## What these scores can help decide\n\nThis report offers a concrete three-node deployment reference, a slow but measured generation path and a transparent small-suite failure history. It can help frame a reproduction: pin the actual artifact and runtime, retain per-task sampling and output caps, record timeout policy, and measure memory alongside the intended request lengths. It does not support a global ranking against cloud models or another quantization format.\n\nThe useful score is therefore the one that keeps its denominator and history. Read 49/50 with the subset and unequal controls; read 18/20 with the initial 14/20 and selected four-task rerun. Keep configured context separate from completed input and approximate occupancy separate from peak memory. Those distinctions allow both readers and AI systems to use the original evidence without turning a limited experiment into a broader capability claim.\n" } ] }, { "id": "qwen-paired-memory-recovery", "title": "Read paired Qwen memory and actual recovery reruns", "needs": [ "articles" ], "purpose": "Keep the unchanged concurrent peak, different 500K peak, production/validation hardware, invalid first ship run and two-event rerun.", "expression": "articles.articles.filter(a => a.id === 'A19')", "expected_output": [ { "id": "A19", "path": "/guides/qwen-flash-next-paired-memory-oom-recovery/", "title": "Qwen Long Concurrency: Paired Memory Peaks and OOM Recovery", "summary": "Read two 250K-request memory controls, a 500K single-request peak, a production OOM and invalidated/repeated recovery tests without mixing hardware or cohorts.", "answer": "A bounded gather lowers the isolated transient and the 500K single-request device peak, but the paired two-250K host-KV cell reports the same +1248 MiB non-pool peak on both trees. Production failure, development recovery and the smaller ship-tree rerun have different conditions.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "nanogenomic", "repository": "nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000", "platform": "github", "revision": "11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2", "document_path": "BENCHMARKS.md", "pinned_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md", "document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "last_commit_at": "2026-10-01T04:27:43Z", "url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000", "report_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "qwen-flash-next-nanogen-memory-recovery" } ], "supplemental_sources": [ { "document_id": "qwen-nanogen-readme", "parent_record_id": "qwen-flash-next-nanogen-memory-recovery", "publisher": "nanogenomic", "repository": "nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000", "revision": "11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2", "document_path": "README.md", "pinned_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/README.md", "document_sha256": "bcfd94276d2bebef681e8a7737444ecec9814d6c73dce9ecf95a0b9aa437a3ab", "last_commit_at": "2026-10-01T04:27:43Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "qwen-nanogen-commits", "parent_record_id": "qwen-flash-next-nanogen-memory-recovery", "publisher": "nanogenomic", "repository": "nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000", "revision": "11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2", "document_path": "COMMIT-MAP.tsv", "pinned_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/COMMIT-MAP.tsv", "document_sha256": "7a6b188d895f6d99fe46ca650832e7d1d8f182aa0b2c59e1442bd7794042fa92", "last_commit_at": "2026-10-01T04:27:43Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" } ], "facts": [], "reported_results": [ { "label": "on-GPU control minimum free VRAM", "metric": "available_GPU_memory", "value": 3414, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 870, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "on-GPU control non-pool peak over idle", "metric": "non_pool_memory_over_idle", "value": 1090, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 871, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "host KV minimum free VRAM", "metric": "available_GPU_memory", "value": 6962, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 870, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "host KV non-pool peak over idle", "metric": "non_pool_memory_over_idle", "value": 1248, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 871, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "524288 FP8 old gather isolated transient", "metric": "kernel_transient_memory", "value": 1024, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "old gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 954, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "524288 FP8 bounded gather isolated transient", "metric": "kernel_transient_memory", "value": 32, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "bounded gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 954, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "500K old gather device peak", "metric": "peak_device_used_memory", "value": 97237, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "old gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 972, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "500K bounded gather device peak", "metric": "peak_device_used_memory", "value": 96093, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "bounded gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 972, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "2x250K host-KV old gather non-pool peak", "metric": "non_pool_memory_over_idle", "value": 1248, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "old gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 981, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "2x250K host-KV bounded gather non-pool peak", "metric": "non_pool_memory_over_idle", "value": 1248, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "bounded gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 981, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "Production OOM failed_allocation", "metric": "failed_allocation", "value": 560, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 639, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L639", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "Production OOM available_GPU_memory", "metric": "available_GPU_memory", "value": 280.94, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 640, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L640", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" }, { "label": "Production OOM process_device_memory", "metric": "process_device_memory", "value": 93.92, "unit": "GiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 642, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L642", "source_record_id": "qwen-flash-next-nanogen-memory-recovery" } ], "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch8 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch8-qwen-memory-hero.webp", "alt": "Qwen: the concurrent peak did not fall. Both trees report +1,248 MiB. This is a paired concurrent delta, not total GPU or host RAM.", "width": 1600, "height": 900, "sha256": "a81ca1581ada8f1ba340bc46f1d4d2c624cccc50866af8d4ae763ebd2b712719", "caption": "Both trees report +1,248 MiB. This is a paired concurrent delta, not total GPU or host RAM." }, "case_reports": [ { "id": "deep-queue-production-OOM", "classification": "source_reported_real_allocator_failure", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 633, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L633", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "downtime_seconds": 774, "error": "torch.OutOfMemoryError in eager prefill; scheduler SIGQUIT and process-tree termination", "monitoring_alert_before_crash_seconds": 53, "admission_consumed_alert": false }, { "id": "recovery-branch-versus-ship", "classification": "source_reported_fault_injection_and_real_pressure", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 999, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L999", "branch": "oom-retract, host KV off", "branch_pool_tokens": [ 451712, 451840 ], "runs": [ { "forced": 39, "real": 6, "recoveries": 45 }, { "forced": 39, "real": 3, "recoveries": 42 } ], "engine_alive": true, "scheduler_suite": { "initial_harness_errors": 9, "initial_error": "runner IndexError", "rerun_passed": 100 }, "ship_tree": { "initial_valid": false, "initial_error": "BatchEncoding not JSON serializable", "initial_injects": 0, "initial_recoveries": 0, "initial_verdict": false, "rerun_injections": 2, "rerun_recoveries": 2, "rerun_all_needles_correct": true, "engine_alive": true, "host_KV_pool_tokens": 1310720 }, "recovery_predicates": [ "overlap enabled", "EXTEND", "TP1", "PP1", "no MLP sync", "no disaggregation", "HiSparse off", "PDMux off", "no diffusion config", "device-wide drain completes" ], "max_retractions": 3, "exhaustion_status": 503, "next_request_status": 200, "decode_OOM_recovered": false, "driver_error_recovered": false, "peer_outputs_bit_identical": false }, { "id": "host-UVA-regression", "classification": "unresolved_nonreproducing_microbenchmark", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 992, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L992", "first_new_ms": [ 341, 376 ], "later_new_ms": [ 4.99, 5.68 ], "later_new_peak_MiB": [ 159, 189 ], "first_touch_explanation_proven": false, "kernel_time_improvement_claim": false }, { "id": "composite-gate-missing-same-bundle-NIAH", "classification": "harness_composition_failure", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 873, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L873", "decode_path_gate": true, "standalone_needle_correct": 9, "standalone_needle_total": 9, "same_bundle_NIAH_cells": 0, "long_context_gate": false, "overall_gate": false, "model_quality_failure_established": false } ], "memory_observations": [ { "label": "on-GPU control minimum free VRAM", "value": 3414, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870" }, { "label": "on-GPU control non-pool peak over idle", "value": 1090, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871" }, { "label": "host KV minimum free VRAM", "value": 6962, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870" }, { "label": "host KV non-pool peak over idle", "value": 1248, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871" }, { "label": "524288 FP8 old gather isolated transient", "value": 1024, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "old gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954" }, { "label": "524288 FP8 bounded gather isolated transient", "value": 32, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "bounded gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954" }, { "label": "500K old gather device peak", "value": 97237, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "old gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972" }, { "label": "500K bounded gather device peak", "value": 96093, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "bounded gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972" }, { "label": "2x250K host-KV old gather non-pool peak", "value": 1248, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "old gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981" }, { "label": "2x250K host-KV bounded gather non-pool peak", "value": 1248, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "bounded gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981" }, { "label": "Production OOM failed_allocation", "value": 560, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L639" }, { "label": "Production OOM available_GPU_memory", "value": 280.94, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L640" }, { "label": "Production OOM process_device_memory", "value": 93.92, "unit": "GiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L642" } ], "long_context_cases": [ { "id": "host-KV-two-mains-turn3", "input_length_source_labels": [ "250K", "200K" ], "additional_subagents": 2, "host_KV_pool_tokens": 1310720, "control_pool_tokens": 451840, "host_KV_TTFT_seconds": [ 1.2, 1 ], "control_TTFT_seconds": [ 22.9, 41.1 ], "host_KV_cache_hit_rate": 0.9949, "control_cache_hit_rate": 0, "control_requests_completed": true, "host_peak_RAM": null, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 833, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L833" } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# Qwen Long Concurrency: Paired Memory Peaks and OOM Recovery\n\n![Qwen: the concurrent peak did not fall. Both trees report +1,248 MiB. This is a paired concurrent delta, not total GPU or host RAM.](/assets/batch8-qwen-memory-hero.webp)\n\n*Both trees report +1,248 MiB. This is a paired concurrent delta, not total GPU or host RAM.*\n\nThe most useful memory result in this Qwen3.8-Flash-Next report is a negative one. A change that cuts an isolated allocation and a 500K single-request peak does **not** reduce the measured peak in its two-request, approximately 250K-per-request production-like cell. Keeping that result prevents a deployment guide from turning a kernel improvement into an unsupported concurrency-capacity promise. Run AI Here has read the original measurements; it has not run these model experiments.\n\n## First separate the two physical cards\n\nThe [maintainer README](https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/README.md) distinguishes a production/reference RTX PRO 6000 Blackwell **Max-Q Workstation**, 300 W, PCIe Gen4 x16 and driver 610.43.02, from a rented **Server Edition**, 600 W, PCIe Gen5 x16 and driver 580.95.05 used for later validation. Both report 97,887 MiB device capacity. The host-KV, bounded-gather and recovery validation belongs to the Server Edition. The September 30 production crash belongs to the Max-Q lane. A shared product-family name does not make their latency or failure margins interchangeable, and no controlled cross-SKU scaling result is established.\n\nThe model is a community NVFP4 Qwen3.8-Flash-Next artifact with FP8 KV and host-resident components. The selected report does not give a complete upstream weight revision for these cells. Its patched SGLang branches and a base prefix are documented, but the exact full executed tree and image digest for every arm are missing. The pinned document revision identifies what was read, rather than silently supplying those execution identities.\n\n## The concurrent memory controls answer two questions\n\nIn [section 3.15b of the original benchmark ledger](https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md), a cold two-by-250K prefill compares on-GPU KV against host KV on the same validation card. The control auto-sizes to a 451,840-token pool; host KV uses 1,310,720. The control pool was not intentionally pinned to match the production lane. This matters because placement and capacity change together.\n\n| During the concurrent cold prefill | On-GPU control | Host KV |\n|---|---:|---:|\n| Minimum free device memory, MiB | 3,414 | 6,962 |\n| Peak non-pool allocation above idle, MiB | 1,090 | 1,248 |\n\nThe host-KV arm has more minimum free VRAM while its non-pool increment is slightly larger. Those observations are compatible because they describe different accounting quantities. Neither column supplies peak whole-host RAM. Subtracting a free-memory observation from an unrelated device capacity or combining different idle baselines would create a result the author did not measure.\n\n![Host KV: concurrent prefill headroom. Cold 2 x 250K prefill: minimum free VRAM, not peak host RAM. Source section 3.15b.](/assets/batch8-qwen-headroom.webp)\n\n*Cold 2 x 250K prefill: minimum free VRAM, not peak host RAM. Source section 3.15b.*\n\nA separate old-versus-bounded-gather A/B keeps the host-KV setup and uses two cold approximately 250K requests. Both trees report **+1,248 MiB non-pool peak over idle**. The author explicitly declines an end-to-end peak improvement for this cell. The likely explanation that another allocation dominates is left as a hypothesis; the report does not isolate that allocation. A planner should therefore retain the equal peak when estimating this workload's headroom.\n\n## Why the 500K result can still improve\n\nSection 3.16 traces full-context prefill gather temporaries that grow with the entire request, even when prefill is chunked. In an isolated 524,288-token FP8 kernel microbenchmark, its own transient allocation falls from 1,024.0 to 32.0 MiB, excluding the output. Twelve cases in each of two runs are bit-exact with zero differing values. That is a kernel-level check, not proof that complete generated answers are deterministic or that every inference path is equivalent.\n\nThe in-engine A/B instead holds a 689,728-token pool and 94,661 MiB idle device usage. At 500,000 input tokens, old and new device-used peaks are 97,237 and 96,093 MiB. All nine needle cases across three lengths and three depths pass in both arms. This supports a lower peak for that single-request sweep, while leaving the paired concurrent no-improvement result intact. The smallest kernel case and the wide-int64 case also show why “always 32 MiB” would overstate the implementation's behavior.\n\n![500K: a different cell does improve. In-engine device peaks at 500,000 tokens. Both arms pass 9/9 needles over the full three-length sweep.](/assets/batch8-qwen-500k-peak.webp)\n\n*In-engine device peaks at 500,000 tokens. Both arms pass 9/9 needles over the full three-length sweep.*\n\nHost-UVA timing has an unresolved reproduction problem. The first new-path run takes 341–376 ms, whereas later runs take 4.99–5.68 ms and report 159–189 MiB peaks. A first-touch explanation is suggested but not proven. The source excludes a host-UVA kernel-time improvement claim, and so does this guide.\n\n## Prefix loss can hurt without an HTTP error\n\nThe same-card two-main-agent test sends histories labelled 250K and 200K, then adds two subagents on turn three. Host KV retains prefixes of 249,984 and 200,000 tokens; the control evicts both. Host-KV first-token times are 1.2 and 1.0 seconds, against 22.9 and 41.1 seconds in the control. Every control request still completes. This is evidence of a cache-capacity failure mode rather than an HTTP failure or a general throughput comparison.\n\nThe source's 95.56 percent multi-turn cached fraction belongs to six conversations growing from about 60K to 80K, not to the 250K–500K main agents. Its two-520K host-KV case runs, but the on-GPU control cell does not execute. Keep that missing comparison instead of inventing a speedup.\n\n## The real production outage happened outside a full pool\n\nOn September 30 at 16:44:17, the Max-Q production lane fails an eager-prefill allocation of 560 MiB with 280.94 MiB free. The process reports 93.92 GiB device usage including non-PyTorch memory; another process uses 676 MiB. The configured static fraction is 0.98 and the pool has 522,880 tokens. The final log has one running request, fourteen queued, 1,485,849 pending tokens, token usage 0.73 and Mamba usage 0.06. Scheduler termination interrupts service for 12 minutes 54 seconds.\n\nThat snapshot shows why a token-pool percentage alone cannot bound every external allocation. It does not prove that power limit or PCIe generation caused the crash, nor establish a universal safe static fraction. The queue alert arrives 53 seconds before failure but does not feed admission. A monitoring signal and an enforced dispatch constraint are separate operational facts.\n\n## Recovery counts must keep their branch and invalid attempts\n\nThe development `oom-retract` battery runs with host KV **off**: one run has 39 injected and six real-pressure OOMs, with 45 recoveries; the second has 39 plus three, with 42. A scheduler suite initially has nine runner errors, then passes 100 tests after a harness correction. These are not forty-five ship-tree recoveries.\n\n![Recovery evidence must keep its sample size. The large branch battery is not a 45-event ship-tree validation. Decode and driver errors remain fatal.](/assets/batch8-qwen-recovery.webp)\n\n*The large branch battery is not a 45-event ship-tree validation. Decode and driver errors remain fatal.*\n\nThe first host-KV ship-tree injection attempt is invalid because request serialization fails: zero injections and zero recoveries occur. The valid rerun has two injections, two recoveries, correct needles and a live engine. Recovery requires the source's narrow overlap/EXTEND/TP1/PP1 conditions, the other exclusion predicates and a completed drain. After three retractions an unfit request gets 503; the next request succeeds. Decode OOM and driver failures stay fatal. Peer outputs are not bit-identical, including ordinary OOM-free repeats.\n\nFinally, a composite long-context gate has no NIAH cells in its own bundle and therefore remains false, despite nine separately passing needles. That is a harness-composition gap, not demonstrated model-quality failure. Useful next evidence would combine the real serving workload, repeated per-cell GPU and host peaks, exact executed artifacts and the assembled quality verdict. Those missing fields remain unknown in the JSON export.\n" } ] }, { "id": "new-spark-score-retractions", "title": "Read Spark deployments, score retractions and build-specific retries", "needs": [ "articles" ], "purpose": "Return full GLM and MiMo guides, preserving historical scores, EXL3 withdrawal, missing current task rerun, exact versions and C1/C8 denominators.", "expression": "articles.articles.filter(a => ['A20','A21','A22'].includes(a.id))", "expected_output": [ { "id": "A20", "path": "/guides/glm53-full-four-spark-tensorfold-deployment/", "title": "Full GLM-5.3 on Four Sparks: Cache Layout, Speed and Fidelity", "summary": "Audit the 753B non-Flash EXL3 TP4 recipe, replicated and shared Q5 cache, million-token retrieval, BF16-reference fidelity and later RDMA fixes.", "answer": "The source runs full GLM-5.3 on four GB10 nodes with EXL3 and native MTP-3. Its two cache layouts trade reported aggregate code throughput for a larger shared window. Peak RAM and a task-accuracy suite are absent; the published quality panel measures quantization fidelity.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "bertholomus", "repository": "bertholomus/glm-5.3-tensorfold-tp4-4xgb10", "platform": "github", "revision": "9a5c1100c68cd8ea832803c120ee63313049fc21", "document_path": "README.md", "pinned_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md", "document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "last_commit_at": "2026-10-09T05:31:45Z", "url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10", "report_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "glm53-full-tp4-exl3" } ], "supplemental_sources": [ { "document_id": "glmfull-speed-method", "parent_record_id": "glm53-full-tp4-exl3", "publisher": "bertholomus", "repository": "bertholomus/glm-5.3-tensorfold-tp4-4xgb10", "revision": "9a5c1100c68cd8ea832803c120ee63313049fc21", "document_path": "bench/tf_bench.py", "pinned_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/bench/tf_bench.py", "document_sha256": "9652bf8a7359414ad451dc05ceee4f4bef6cfcf489e65334e9bec1e7895192b2", "last_commit_at": "2026-10-09T05:31:45Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glmfull-needle-method", "parent_record_id": "glm53-full-tp4-exl3", "publisher": "bertholomus", "repository": "bertholomus/glm-5.3-tensorfold-tp4-4xgb10", "revision": "9a5c1100c68cd8ea832803c120ee63313049fc21", "document_path": "bench/tf_needle.py", "pinned_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/bench/tf_needle.py", "document_sha256": "cdf8ab15fc7ec88073a714696b561f2e7a292ac9a0490571935bbe2cc933a4ab", "last_commit_at": "2026-10-09T05:31:45Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glmfull-greedy-method", "parent_record_id": "glm53-full-tp4-exl3", "publisher": "bertholomus", "repository": "bertholomus/glm-5.3-tensorfold-tp4-4xgb10", "revision": "9a5c1100c68cd8ea832803c120ee63313049fc21", "document_path": "bench/tf_greedy.py", "pinned_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/bench/tf_greedy.py", "document_sha256": "834c9f1e73af9431b585adf00aa3f14254d8713b638a4fa95c36b6a2074dbf37", "last_commit_at": "2026-10-09T05:31:45Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" } ], "facts": [], "reported_results": [ { "label": "replicated Q5 four code streams", "metric": "aggregate_output_rate", "value": 82, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "Q5 MLA latent cache; FP8 indexer", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": "2026-10-04", "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": 4, "context_limit_tokens": 65536, "cache_state": "replicated", "metric_denominator": "source reports HTTP prompt time included; release concurrent raw usage/harness not retained", "token_accounting_verified": false, "available_serial_stream_probe_counts_nonempty_SSE_chunks": true, "release_was_computed_by_that_probe": null, "value_is_approximate": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 57, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L57", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "DCP4 shared Q5 four code streams", "metric": "aggregate_output_rate", "value": 64, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "Q5 MLA latent cache; FP8 indexer", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": "2026-10-04", "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": 4, "context_limit_tokens": 1048576, "cache_state": "four streams share one DCP4 pool", "metric_denominator": "source reports HTTP prompt time included; release concurrent raw usage/harness not retained", "token_accounting_verified": false, "available_serial_stream_probe_counts_nonempty_SSE_chunks": true, "release_was_computed_by_that_probe": null, "value_is_approximate": true }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 68, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L68", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "32K-labelled prefill", "metric": "prefill_rate", "value": 402, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "Q5 MLA latent cache; FP8 indexer", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": "2026-10-04", "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": 1, "input_length_source_label": "32K", "context_limit_tokens": 1048576, "cache_state": null, "metric_denominator": "source HTTP prefill measurement, exact prompt count unreported" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 71, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L71", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "forward KL", "metric": "KL_reference_to_quant", "value": 0.109, "unit": "nats", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "no cache quantization", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": null, "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": null, "quality_panel": "wikitext-2 test, 32 x 2048 = 65536 positions", "activation_dtype": "BF16", "reference": "BF16 full model", "cache_state": null, "quality_is_task_accuracy": false, "quality_arm": "forward KL" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 167, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L167", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "reverse KL", "metric": "KL_quant_to_reference", "value": 0.123, "unit": "nats", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "no cache quantization", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": null, "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": null, "quality_panel": "wikitext-2 test, 32 x 2048 = 65536 positions", "activation_dtype": "BF16", "reference": "BF16 full model", "cache_state": null, "quality_is_task_accuracy": false, "quality_arm": "reverse KL" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 168, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L168", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "top-1 agreement", "metric": "top1_agreement", "value": 90.2, "unit": "percent", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "no cache quantization", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": null, "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": null, "quality_panel": "wikitext-2 test, 32 x 2048 = 65536 positions", "activation_dtype": "BF16", "reference": "BF16 full model", "cache_state": null, "quality_is_task_accuracy": false, "quality_arm": "top-1 agreement" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 169, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L169", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "quant perplexity", "metric": "perplexity", "value": 2.911, "unit": "perplexity", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "no cache quantization", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": null, "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": null, "quality_panel": "wikitext-2 test, 32 x 2048 = 65536 positions", "activation_dtype": "BF16", "reference": "BF16 full model", "cache_state": null, "quality_is_task_accuracy": false, "quality_arm": "quant perplexity" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 170, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L170", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "BF16 perplexity", "metric": "perplexity", "value": 2.75, "unit": "perplexity", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "no cache quantization", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": null, "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": null, "quality_panel": "wikitext-2 test, 32 x 2048 = 65536 positions", "activation_dtype": "BF16", "reference": "BF16 full model", "cache_state": null, "quality_is_task_accuracy": false, "quality_arm": "BF16 perplexity" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 170, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L170", "source_record_id": "glm53-full-tp4-exl3" }, { "label": "served engine prompt-path KL", "metric": "KL_reference_to_engine", "value": 0.111, "unit": "nats", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "served path; panel-specific cache details not retained", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": null, "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null, "concurrency": null, "quality_panel": "wikitext-2 test, 32 x 2048 = 65536 positions", "activation_dtype": "BF16", "reference": "BF16 full model", "cache_state": null, "quality_is_task_accuracy": false, "quality_arm": "served engine" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 173, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L173", "source_record_id": "glm53-full-tp4-exl3" } ], "conditions": { "model": "GLM-5.3 (full 753B, not Flash)", "model_artifact": "bertholomus/GLM-5.3-EXL3-3.0bpw", "model_revision": null, "engine": "TensorFold 0.6.0 + glm-dsa-tp4 branch", "runtime_execution_revision": null, "measured_runtime_short": "3eb35dd", "later_branch_head_short": "ddfad35", "executed_image_digest": null, "system": "four DGX Spark GB10, 128 GB unified memory each, direct dual-port 200G RoCE", "node_count": 4, "parallelism": "TP4", "weight_precision": "EXL3 3.04 bpw whole-model average; exllamav3 1.5.3", "kv_dtype": "Q5 MLA latent cache; FP8 indexer", "draft": "native MTP-3", "vision": false, "DFlash": false, "experiment_date": "2026-10-04", "temperature": 0, "thinking": null, "input_tokens": null, "output_tokens": null, "host_peak_RAM": null, "repetitions": null }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch8 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch8-glmfull-hero.webp", "alt": "Full GLM-5.3: four Sparks, two cache layouts. Source-reported HTTP rates include prompt time; exact concurrent token accounting is not retained.", "width": 1600, "height": 900, "sha256": "161d7c2e8521aa97f46e84b13468dd2382ed809d5065dac1cd67c11a5509f24b", "caption": "Source-reported HTTP rates include prompt time; exact concurrent token accounting is not retained." }, "case_reports": [ { "classification": "source_reported_RDMA_completion_collision", "old_error": "ibv_post_recv failed", "fix_short": "cad0d2a", "measured_release_short": "3eb35dd", "later_head_short": "ddfad35", "full_execution_revision": null, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 35, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L35" } ], "quality_comparisons": [ { "panel": "32x2048 wikitext-2", "BF16_activations": true, "cache_quantization": false, "forward_KL": 0.109, "reverse_KL": 0.123, "top1_percent": 90.2, "quant_PPL": 2.911, "BF16_PPL": 2.75, "task_accuracy": null } ], "memory_observations": [ { "kind": "shape-derived weights per rank, not peak", "value": 69.86, "unit": "GiB", "is_measured_peak": false, "whole_system_peak": null, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 121, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L121" } ], "long_context_cases": [ { "actual_input_tokens": 1039064, "needle_found": true, "independent_repeats": null, "concurrent_full_window_requests_proven": false, "source_document_id": "glmfull-tp4-readme", "source_document_sha256": "088db2d55fbb6aecc2cc6576b3db9a26e70a13a2b5d89b83ad1cb0b54b03a6b4", "source_line": 71, "evidence_url": "https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md#L71" } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# Full GLM-5.3 on Four Sparks: Cache Layout, Speed and Fidelity\n\n![Full GLM-5.3: four Sparks, two cache layouts. Source-reported HTTP rates include prompt time; exact concurrent token accounting is not retained.](/assets/batch8-glmfull-hero.webp)\n\n*Source-reported HTTP rates include prompt time; exact concurrent token accounting is not retained.*\n\nFull GLM-5.3 is a different deployment problem from GLM-5.3-Flash. The [BertholomusAI four-GB10 recipe](https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/README.md) targets the 753B non-Flash model, all 79 layers plus its MTP layer, using EXL3 and TensorFold. Its value is a documented route to fitting and serving that full model, with explicit cache layouts and a fidelity panel. It does not provide a general model leaderboard, a production peak-memory trace or a claim that four long jobs each occupy the entire configured window. Run AI Here has not reproduced its GPU measurements.\n\n## The hardware and quantization contract\n\nThe report uses four DGX Spark/GB10 nodes with 128 GB unified memory each, one tensor-parallel rank per node, direct RoCE links and both 200G ports. Its runtime is TensorFold 0.6.0 plus the author's `glm-dsa-tp4` branch. The source credits TensorFold's original author separately from the model-specific recipe. Substituting a switched fabric, different rank layout or another engine would be another deployment requiring its own evidence.\n\nThe quantization uses exllamav3 1.5.3: routed experts are 3-bit, attention/shared experts/dense layers are promoted, the MTP experts use 4 bits and the output head uses 6. The reported whole-model average is 3.04 bits per weight, across 39 shards totalling 292.7 GB. A “3.0 bpw” artifact name is therefore not a statement that every tensor has identical precision. The exact selected checkpoint revision is not retained in the recipe, so the artifact name and null revision remain separate fields.\n\nThe source derives about 69.86 GiB weights per rank from its real configuration. That is a fit calculation, not a sampled peak. It excludes the complete inference working set, graph captures, cache buffers and operating-system demands. File GB, weight GiB and peak unified-memory usage should not be treated as interchangeable capacity numbers. Peak whole-system RAM and a time-series sampling method are absent.\n\n## Keep the release tree distinct from its later fixes\n\nOctober 4 release measurements are labelled engine prefix `3eb35dd`. The document's later branch head is `ddfad35`, with an RDMA completion-matching correction and graph prewarm work on top. Only short executed prefixes are provided for this table; the document's full Git revision does not fill that gap. Both identities are retained so a reader can identify the measurement generation before trying a newer recipe.\n\n![Measured release and later fixes differ. The release speed table remains attributed to 3eb35dd, not silently relabelled as the later branch head.](/assets/batch8-glmfull-identity.webp)\n\n*The release speed table remains attributed to 3eb35dd, not silently relabelled as the later branch head.*\n\nThe RDMA incident involves matching queue-pair numbers across two devices, where a collision can produce `ibv_post_recv failed`. The later fix scopes matching to the device. Another change captures rounds and draft-step graphs during warm-up, removing work from the first concurrent burst. The source reports finite greedy/concurrent equality checks after those changes. It explicitly leaves the original release table unchanged; this guide does not relabel all old numbers as a fresh run of the new head.\n\n## Two cache layouts make different promises\n\nConfiguration A uses replicated Q5 cache, a 65,536-token limit per stream and four streams. It reports 82 aggregate code tokens/s, with 0.59–0.60-second first-token times, plus 71–73 aggregate chat tokens/s. These are source HTTP observations, not per-user rates. The prompts, cache arrangement and completion behavior matter to any comparison.\n\nConfiguration B uses Q5 cache with decode context parallelism over four ranks and a shared 1,048,576-token pool. Four code streams report approximately 64 aggregate tokens/s; chat reports 56. Windows are extents in one pool. This trades a different cache arrangement for a larger configured context. It does not grant four independent million-token allocations. Without raw input and output counts for every concurrent cell, the report supports its own conditional observations rather than a normalized throughput claim.\n\nThe source uses GLM's native MTP layer with three draft steps, and no DFlash. Vision is not implemented in this release. Those deployment choices stay attached to the speeds. A separate Flash model with another drafter, parser or visual path would not be a matched performance control.\n\n## Audit the token-rate denominator before comparing engines\n\nThe README describes its release numbers as HTTP measurements with prompt time included. The available [serial streaming probe](https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/bench/tf_bench.py) counts nonempty SSE chunks and derives a post-first-chunk rate. Chunks are not guaranteed to contain one tokenizer token, and that script does not independently validate them against usage counts. The exact concurrent harness and raw usage underlying the release table are not retained in this repository. It is not established that the concurrent release table was computed by this particular serial helper.\n\nThese limitations should travel with the values, rather than becoming either an invented corrected speed or an accusation that every reported rate uses the helper's denominator. The separate [greedy-output script](https://github.com/bertholomus/glm-5.3-tensorfold-tp4-4xgb10/blob/9a5c1100c68cd8ea832803c120ee63313049fc21/bench/tf_greedy.py) does read completion usage and wall time for its own requests. That does not retrospectively establish how every other table was calculated. A future reproducible comparison should retain synchronized group wall time, tokenizer or usage output totals, warm-up policy and both TTFT and decode definitions.\n\n## The quality panel measures fidelity to BF16\n\nThe source's quantization panel uses wikitext-2 test text, 32 segments of 2,048 tokens, BF16 activations and no cache quantization. Forward KL from BF16 to quantized probabilities is 0.109, reverse KL is 0.123 and top-1 agreement is 90.2 percent. Perplexity is 2.911 for the quantized model versus 2.750 for BF16. These are specific probability/text-panel measurements, not percentages of correctly answered tasks.\n\n![Quantization fidelity: one text panel. Panel perplexity is not MMLU, GSM8K or a general task score; no benchmark ranking is derived.](/assets/batch8-glmfull-perplexity.webp)\n\n*Panel perplexity is not MMLU, GSM8K or a general task score; no benchmark ranking is derived.*\n\nA served-engine prompt-path check reports KL 0.111 against stored BF16 logits on the same panel. Another longer text panel exercises sparse attention and has a different KL; a separate cache-quantization comparison answers yet another question. Combining weight quantization, cache quantization and served-engine differences into one “accuracy loss” would erase the controls. The selected JSON preserves each comparison's reference and scope. No MMLU, GSM8K or coding-task score is inferred from a small KL or a finite greedy equality check.\n\n## The long-window evidence is useful and finite\n\nThe report finds a needle at 1,039,064 actual input tokens. It also reports 402 prefill tokens/s at a source-labelled 32K input and approximately 76 seconds to first token near 31K. Those rounded input labels are not exact token counts. A planner interested in long documents should budget prefill latency separately from the better short-request decode rate.\n\n![A configured window is not four full-window jobs. A per-rank weight estimate excludes cache, graphs, working memory and system services.](/assets/batch8-glmfull-capacity.webp)\n\n*A per-rank weight estimate excludes cache, graphs, working memory and system services.*\n\nThe million-token result establishes a completed retrieval case in this setup. It does not establish multi-hop accuracy, repeated long-context stability or simultaneous full-window requests. The repository's needle helper and finite output-equality probes give useful inspection points, but missing repetitions, model revision, exact concurrent accounting and peak memory remain missing. This recipe is most useful when those boundaries guide the next experiment: hold the artifact and runtime fixed, record the fabric and cache layout, then capture latency, completion correctness and memory over the same workload interval.\n" }, { "id": "A21", "path": "/guides/glm53-flash-scores-router-bias-retraction/", "title": "GLM Flash Scores: Historical NVFP4 and the EXL3 Retraction", "summary": "Separate a full historical MMLU run and small task subsets from the corrected EXL3 router stack, whose old task scores were withdrawn and await reruns.", "answer": "The September NVFP4 report and October EXL3 correction are different deployment lines. The EXL3 author withdrew its old task scores after finding an unloaded router bias. Current evidence is a same-quant probability check; it does not revalidate old task scores or import the NVFP4 task panel.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "platform": "github", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "docs/06-benchmarks.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md", "document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "last_commit_at": "2026-09-09T21:23:55Z", "url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "report_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "glm53-flash-nvfp4-tp3-scores" }, { "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "platform": "github", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "tracks/tp3-main/README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/tracks/tp3-main/README.md", "document_sha256": "6cb79207b37211c1542b98dfe26408e7d0b741133be67286db7838c48976223f", "last_commit_at": "2026-10-16T01:23:58Z", "url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "report_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/tracks/tp3-main/README.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "glm53-flash-exl3-router-correction" } ], "supplemental_sources": [ { "document_id": "glm-tp3-readme", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/README.md", "document_sha256": "5912e935333340dbd77ecd01cec8c7e207990bc6f446341f741bb3bd66d6cf4b", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-memory", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "docs/05-memory-ladder.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/05-memory-ladder.md", "document_sha256": "29874450acd65f1e6614bd3fc04fe0815ef8b9036007867b8446e63ebb4092de", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-mmlu", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/mmlu/mmlu-full-lm-eval-results.json", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json", "document_sha256": "86ed1c5586cfe259e7ff438fff2baf1df2dd2ebc069b3db5bbce0763c9ef03c8", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-tool", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/tool-eval-bench/tool-eval-bench-8-trials.json", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/tool-eval-bench/tool-eval-bench-8-trials.json", "document_sha256": "7a46e4d7b52956a0fb0662838081b526e7ceeb30e14e553648227bade4dc6df5", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-needle", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/needle/needle-in-a-haystack-report.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/needle/needle-in-a-haystack-report.md", "document_sha256": "cceef7123bbb9e9d34c2c6a44bc00055b81f896cbf51977053766e770c30ae3d", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-results", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/README.md", "document_sha256": "15388126cd782399e618a52b5cbcd2c5af46f7de347f6903e4705d719f84b037", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-readme", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/README.md", "document_sha256": "a399f1dd57e66933e4b947eb4f14212815dc0526deee69942624759b178038d0", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-kl", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "results/main-stack/kl-health.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md", "document_sha256": "9fcb41cb68098fe0e982f727f6c57b86656748fd41bbd6a0f64dfc794dbb2ba1", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-gates", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "results/main-stack/quality-gates.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/quality-gates.md", "document_sha256": "9d0387be075bd6c214ab8e9b2bb70a46577b5052884c06c0db02d1869b295f28", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-stack", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "docs/20-main-stack.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/docs/20-main-stack.md", "document_sha256": "b7ef6f4c82dd1bfa693452cb1fcfb57c8f3270455ca894b667e24048cc20786d", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-readme", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/README.md", "document_sha256": "5912e935333340dbd77ecd01cec8c7e207990bc6f446341f741bb3bd66d6cf4b", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-memory", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "docs/05-memory-ladder.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/05-memory-ladder.md", "document_sha256": "29874450acd65f1e6614bd3fc04fe0815ef8b9036007867b8446e63ebb4092de", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-mmlu", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/mmlu/mmlu-full-lm-eval-results.json", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json", "document_sha256": "86ed1c5586cfe259e7ff438fff2baf1df2dd2ebc069b3db5bbce0763c9ef03c8", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-tool", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/tool-eval-bench/tool-eval-bench-8-trials.json", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/tool-eval-bench/tool-eval-bench-8-trials.json", "document_sha256": "7a46e4d7b52956a0fb0662838081b526e7ceeb30e14e553648227bade4dc6df5", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-needle", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/needle/needle-in-a-haystack-report.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/needle/needle-in-a-haystack-report.md", "document_sha256": "cceef7123bbb9e9d34c2c6a44bc00055b81f896cbf51977053766e770c30ae3d", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-results", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/README.md", "document_sha256": "15388126cd782399e618a52b5cbcd2c5af46f7de347f6903e4705d719f84b037", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-readme", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/README.md", "document_sha256": "a399f1dd57e66933e4b947eb4f14212815dc0526deee69942624759b178038d0", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-kl", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "results/main-stack/kl-health.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md", "document_sha256": "9fcb41cb68098fe0e982f727f6c57b86656748fd41bbd6a0f64dfc794dbb2ba1", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-gates", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "results/main-stack/quality-gates.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/quality-gates.md", "document_sha256": "9d0387be075bd6c214ab8e9b2bb70a46577b5052884c06c0db02d1869b295f28", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-stack", "parent_record_id": "glm53-flash-exl3-router-correction", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "docs/20-main-stack.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/docs/20-main-stack.md", "document_sha256": "b7ef6f4c82dd1bfa693452cb1fcfb57c8f3270455ca894b667e24048cc20786d", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" } ], "facts": [], "reported_results": [ { "label": "Tool-eval eight-trial mean", "metric": "tool_eval_score", "value": 87.8, "unit": "points", "statistic": "mean across eight trials; source SD 0.9, source CI [87.1,88.2]", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "tool-eval-bench 2.6.1.dev39+gd3352edf5", "scenarios": 88, "trials": 8, "seed": 42, "max_turns": 8, "concurrency": 1, "source_trial_SD": 0.9, "source_CI95": [ 87.1, 88.2 ], "CI_method_independently_verified": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 47, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L47", "source_record_id": "glm53-flash-nvfp4-tp3-scores" }, { "label": "Tool-eval Pass^8", "metric": "all_trials_pass_rate", "value": 76.1, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "tool-eval-bench", "scenarios": 88, "trials": 8, "concurrency": 1 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 56, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L56", "source_record_id": "glm53-flash-nvfp4-tp3-scores" }, { "label": "GSM8K 200-question subset", "metric": "accuracy", "value": 94, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "GSM8K", "subset_size": 200, "full_test_size": 1319, "correct": 188, "shots": 8, "CoT": true, "concurrency": null }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 197, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L197", "source_record_id": "glm53-flash-nvfp4-tp3-scores" }, { "label": "IFEval prompt strict, custom evaluator", "metric": "accuracy", "value": 78.9, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "IFEval", "prompts": 541, "correct": 427, "concurrency": null, "evaluator": "custom; not independently cross-checked", "proven_lower_bound": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 155, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L155", "source_record_id": "glm53-flash-nvfp4-tp3-scores" }, { "label": "IFEval instruction level, custom evaluator", "metric": "accuracy", "value": 85.1, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "IFEval", "instructions": 834, "correct": 710, "concurrency": null, "evaluator": "custom; not independently cross-checked", "proven_lower_bound": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 156, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L156", "source_record_id": "glm53-flash-nvfp4-tp3-scores" }, { "label": "Needle four sizes x five depths", "metric": "retrieval_correct_count", "value": 20, "unit": "correct probes", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "single-fact NIAH", "correct": 20, "total": 20, "max_actual_input_tokens": 997952, "concurrency": null, "initial_timeout_seconds": 120, "rerun_timeout_seconds": 3600, "identical_content_seed_on_rerun": true, "multihop_validated": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 221, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L221", "source_record_id": "glm53-flash-nvfp4-tp3-scores" }, { "label": "Full MMLU exact accuracy", "metric": "accuracy", "value": 0.8594217347956131, "unit": "fraction", "statistic": "single full MMLU, harness standard error 0.0028276391244761038", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": null, "thinking": null, "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "MMLU", "subset_size": 14042, "subjects": 57, "shots": 0, "scoring": "loglikelihood, no generation or sampling", "harness": "lm-eval 0.4.9", "concurrency": 8, "timeout_seconds": 36000, "max_retries": 6, "accuracy_stderr": 0.0028276391244761038, "stderr_is_SD": false, "initial_harness_timeout_at_fraction": 0.81, "engine_errors_reported": 0, "rerun_same_content": true }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-mmlu", "source_document_sha256": "86ed1c5586cfe259e7ff438fff2baf1df2dd2ebc069b3db5bbce0763c9ef03c8", "source_line": 4, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json#L4", "source_record_id": "glm53-flash-nvfp4-tp3-scores" }, { "label": "Pre-fix assistant KL", "metric": "KL_reference_to_engine", "value": 0.1033, "unit": "nats", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "turboderp/GLM-5.3-Flash-exl3", "model_revision": "2a30229e67012798ba9f0cd832bb78abf4c363d5", "engine": "vLLM 0.30.1rc1.dev709+g21d93d0d8 plus cuda-exl3 and 14 patches", "base_engine_revision": "21d93d0d8c0e9627900020382bfce4730e61cab7", "runtime_execution_revision": null, "production_plugin_revision": "448f1d6b8964b96e7d4e299494538bfc240149ef", "executed_image_digest": null, "production_image_id": "sha256:8c4f560d92ba190ed84983bdf12e9975c5b4f1ec6dc380b5b93041b2a19ac261", "system": "three DGX Spark GB10", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "EXL3 4.05 bpw full scope", "kv_dtype": "FP8", "experiment_date": "2026-10-07 to 2026-10-08", "concurrency": null, "input_tokens": 21072, "scored_positions": 21066, "assistant_positions": 6601, "quality_panel": "six author-written texts; text set and per-position logits not published", "reference": "exllamav3 1.5.4 running SAME quantized checkpoint, not BF16", "metric_denominator": "assistant-token KL, reference top512 + remainder bin; lower-bound estimate", "temperature": null, "thinking": "reference panel has reasoning effort low / clear_thinking", "cache_state": "prefill target log probabilities, before DFlash proposals", "memory_utilization": 0.75, "host_peak_RAM": null, "current_task_benchmark_score": null, "old_EXL3_task_scores_withdrawn": true, "TP2_corrected_or_tested": false, "measured_build": "a4f", "production_perplexity": null, "pre_fix_comparison_is_retracted_task_score": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-exl3-kl", "source_document_sha256": "9fcb41cb68098fe0e982f727f6c57b86656748fd41bbd6a0f64dfc794dbb2ba1", "source_line": 40, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md#L40", "source_record_id": "glm53-flash-exl3-router-correction" }, { "label": "Production assistant KL", "metric": "KL_reference_to_engine", "value": 0.00828, "unit": "nats", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "turboderp/GLM-5.3-Flash-exl3", "model_revision": "2a30229e67012798ba9f0cd832bb78abf4c363d5", "engine": "vLLM 0.30.1rc1.dev709+g21d93d0d8 plus cuda-exl3 and 14 patches", "base_engine_revision": "21d93d0d8c0e9627900020382bfce4730e61cab7", "runtime_execution_revision": null, "production_plugin_revision": "448f1d6b8964b96e7d4e299494538bfc240149ef", "executed_image_digest": null, "production_image_id": "sha256:8c4f560d92ba190ed84983bdf12e9975c5b4f1ec6dc380b5b93041b2a19ac261", "system": "three DGX Spark GB10", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "EXL3 4.05 bpw full scope", "kv_dtype": "FP8", "experiment_date": "2026-10-07 to 2026-10-08", "concurrency": null, "input_tokens": 21072, "scored_positions": 21066, "assistant_positions": 6601, "quality_panel": "six author-written texts; text set and per-position logits not published", "reference": "exllamav3 1.5.4 running SAME quantized checkpoint, not BF16", "metric_denominator": "assistant-token KL, reference top512 + remainder bin; lower-bound estimate", "temperature": null, "thinking": "reference panel has reasoning effort low / clear_thinking", "cache_state": "prefill target log probabilities, before DFlash proposals", "memory_utilization": 0.75, "host_peak_RAM": null, "current_task_benchmark_score": null, "old_EXL3_task_scores_withdrawn": true, "TP2_corrected_or_tested": false, "measured_build": "a4i", "production_perplexity": null, "pre_fix_comparison_is_retracted_task_score": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-exl3-kl", "source_document_sha256": "9fcb41cb68098fe0e982f727f6c57b86656748fd41bbd6a0f64dfc794dbb2ba1", "source_line": 42, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md#L42", "source_record_id": "glm53-flash-exl3-router-correction" }, { "label": "Bias-fix assistant perplexity", "metric": "perplexity", "value": 2.222, "unit": "perplexity", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "turboderp/GLM-5.3-Flash-exl3", "model_revision": "2a30229e67012798ba9f0cd832bb78abf4c363d5", "engine": "vLLM 0.30.1rc1.dev709+g21d93d0d8 plus cuda-exl3 and 14 patches", "base_engine_revision": "21d93d0d8c0e9627900020382bfce4730e61cab7", "runtime_execution_revision": null, "production_plugin_revision": "448f1d6b8964b96e7d4e299494538bfc240149ef", "executed_image_digest": null, "production_image_id": "sha256:8c4f560d92ba190ed84983bdf12e9975c5b4f1ec6dc380b5b93041b2a19ac261", "system": "three DGX Spark GB10", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "EXL3 4.05 bpw full scope", "kv_dtype": "FP8", "experiment_date": "2026-10-07 to 2026-10-08", "concurrency": null, "input_tokens": 21072, "scored_positions": 21066, "assistant_positions": 6601, "quality_panel": "six author-written texts; text set and per-position logits not published", "reference": "exllamav3 1.5.4 running SAME quantized checkpoint, not BF16", "metric_denominator": "assistant-token KL, reference top512 + remainder bin; lower-bound estimate", "temperature": null, "thinking": "reference panel has reasoning effort low / clear_thinking", "cache_state": "prefill target log probabilities, before DFlash proposals", "memory_utilization": 0.75, "host_peak_RAM": null, "current_task_benchmark_score": null, "old_EXL3_task_scores_withdrawn": true, "TP2_corrected_or_tested": false, "measured_build": "a4g", "production_perplexity": null, "pre_fix_comparison_is_retracted_task_score": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-exl3-kl", "source_document_sha256": "9fcb41cb68098fe0e982f727f6c57b86656748fd41bbd6a0f64dfc794dbb2ba1", "source_line": 41, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md#L41", "source_record_id": "glm53-flash-exl3-router-correction" } ], "conditions": { "historical_NVFP4": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null }, "corrected_EXL3": { "model": "GLM-5.3-Flash", "model_artifact": "turboderp/GLM-5.3-Flash-exl3", "model_revision": "2a30229e67012798ba9f0cd832bb78abf4c363d5", "engine": "vLLM 0.30.1rc1.dev709+g21d93d0d8 plus cuda-exl3 and 14 patches", "base_engine_revision": "21d93d0d8c0e9627900020382bfce4730e61cab7", "runtime_execution_revision": null, "production_plugin_revision": "448f1d6b8964b96e7d4e299494538bfc240149ef", "executed_image_digest": null, "production_image_id": "sha256:8c4f560d92ba190ed84983bdf12e9975c5b4f1ec6dc380b5b93041b2a19ac261", "system": "three DGX Spark GB10", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "EXL3 4.05 bpw full scope", "kv_dtype": "FP8", "experiment_date": "2026-10-07 to 2026-10-08", "concurrency": null, "input_tokens": 21072, "scored_positions": 21066, "assistant_positions": 6601, "quality_panel": "six author-written texts; text set and per-position logits not published", "reference": "exllamav3 1.5.4 running SAME quantized checkpoint, not BF16", "metric_denominator": "assistant-token KL, reference top512 + remainder bin; lower-bound estimate", "temperature": null, "thinking": "reference panel has reasoning effort low / clear_thinking", "cache_state": "prefill target log probabilities, before DFlash proposals", "memory_utilization": 0.75, "host_peak_RAM": null, "current_task_benchmark_score": null, "old_EXL3_task_scores_withdrawn": true, "TP2_corrected_or_tested": false } }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch8 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch8-glmflash-hero.webp", "alt": "GLM Flash: keep the scoring line attached. No NVFP4 score is assigned to the corrected EXL3 deployment. No cross-stack ranking.", "width": 1600, "height": 900, "sha256": "34148f9fadc7ad5f415ad084beb2014c1c1e335782d5c6d2d5d81571894295da", "caption": "No NVFP4 score is assigned to the corrected EXL3 deployment. No cross-stack ranking." }, "case_reports": [ { "id": "NVFP4-harness-only-reruns", "classification": "source_reported_harness_timeouts", "MMLU_initial_progress_fraction": 0.81, "MMLU_initial_error": "aiohttp total timeout includes queue", "MMLU_rerun_timeout_seconds": 36000, "needle_initial_timeout_seconds": 120, "needle_rerun_timeout_seconds": 3600, "same_content_and_seed": true, "reported_engine_errors": 0, "initial_partial_not_full_score": true, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 35, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L35" }, { "id": "EXL3-router-bias-missing", "classification": "source_reported_correctness_bug_and_retraction", "plugin_pre_fix_short": "754421f or earlier", "missing_parameter": "e_score_correction_bias", "source_scope": "EXL3 plugin line only; no evidence that separate NVFP4 line shares defect", "old_task_scores_withdrawn": true, "current_task_score": null, "source_document_id": "glm-exl3-readme", "source_document_sha256": "a399f1dd57e66933e4b947eb4f14212815dc0526deee69942624759b178038d0", "source_line": 18, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/README.md#L18" } ], "quality_comparisons": [ { "id": "NVFP4-historical-task-panel", "record_id": "glm53-flash-nvfp4-tp3-scores", "task_scores_current_EXL3": false, "task_scores_withdrawn_by_EXL3_notice": false, "source_scope": "separate NVFP4 repository, September 3 measurements", "tool_trial_scores": [ 88, 88, 86, 87, 88, 88, 88, 89 ], "tool_mean": 87.8, "source_SD": 0.9, "source_CI95": [ 87.1, 88.2 ], "MMLU_accuracy": 0.8594217347956131, "MMLU_standard_error": 0.0028276391244761038, "IFEval_evaluator_crosschecked": false, "proven_IFEval_lower_bound": false }, { "id": "EXL3-retraction-and-rerun-gap", "record_id": "glm53-flash-exl3-router-correction", "withdrawn_on": "2026-10-08", "old_task_scores_withdrawn": true, "current_task_accuracy": null, "current_task_benchmarks_rerun": false, "router_bias_loaded_after_fix": true, "production_KL": 0.00828, "production_PPL": null, "bias_fix_PPL": 2.222, "reference_PPL": 2.224, "BF16_reference": false, "public_text_set_available": false, "full_per_position_logits_public": false, "TP2_fixed_or_validated": false } ], "memory_observations": [ { "kind": "historical operational free host memory, not peak", "value_range": [ 6.4, 10 ], "unit": "GB, source approximate", "historical_NVFP4_only": true, "peak_RAM": null, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 28, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L28" } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# GLM Flash Scores: Historical NVFP4 and the EXL3 Retraction\n\n![GLM Flash: keep the scoring line attached. No NVFP4 score is assigned to the corrected EXL3 deployment. No cross-stack ranking.](/assets/batch8-glmflash-hero.webp)\n\n*No NVFP4 score is assigned to the corrected EXL3 deployment. No cross-stack ranking.*\n\nA deployment's scores need a version history just as much as its launch command does. Two original GLM-5.3-Flash reports illustrate why: a September NVFP4 task campaign and an October EXL3 correctness investigation. Both use three DGX Sparks, but they differ in quantization, plugin, engine and measurement purpose. The EXL3 author withdrew its old task scores after finding a missing router bias. Reading both reports is useful only if their boundaries survive the summary. Run AI Here has not rerun either deployment or assigned them a combined score.\n\n## The historical NVFP4 campaign has its own identity\n\nThe [September 3 benchmark ledger](https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md) describes a continuous 09:08–18:08 UTC campaign using three GB10 nodes, TP3 plus expert parallelism, Marlin MoE, FP8 KV and DFlash2 k7. The image is `harem/glm53-lil:t10`, with a short engine identity. The configured context ceiling is **1,000,000 decimal tokens**, max sequences eight and batched tokens 2,048. Memory utilization for these benchmark runs is **0.85**, rather than the later production 0.88 setting.\n\nThe [NVFP4 project README](https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/README.md) says this line stopped maintenance on September 6 and moved to EXL3. The old measurements remain historical observations of that stack. Their checkpoint revision and executed image digest are not pinned, so neither is manufactured from the documentation commit. Most generation tests use temperature zero with low reasoning effort; MMLU is the explicit exception because it performs loglikelihood scoring without generation or sampling.\n\n## Full MMLU is different from a small generation subset\n\nThe saved [raw MMLU result](https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json) gives accuracy **0.8594217347956131** and standard error **0.0028276391244761038**. This is the full 14,042-question, 57-subject, zero-shot run under lm-eval 0.4.9. The uncertainty field is a harness standard error, not a repeated-trial standard deviation or a confidence interval. Replacing that meaning with a generic plus/minus label would lose useful statistical information.\n\nThe harness uses concurrency eight, a 36,000-second timeout and six retries. An initial attempt stalls around 81 percent because aiohttp's total timeout includes queued work. The repeated full run keeps the content and reports no engine errors. The failed harness attempt is not a measured zero-accuracy model run, and raising the request timeout is not an engine OOM fix. The final result and the reason for the rerun both remain queryable.\n\n![The check that exposed the routing defect. Six texts, 21,066 scored positions; 6,601 assistant positions. Reference is not BF16.](/assets/batch8-glmflash-kl.webp)\n\n*Six texts, 21,066 scored positions; 6,601 assistant positions. Reference is not BF16.*\n\nThe GSM8K result is instead 188 correct out of 200 selected questions, with eight-shot chain-of-thought prompts and low effort. It is 94.0 percent on that subset, not the full 1,319-question test and not a matched comparison to a zero-shot, thinking-off deployment. Different prompting and answer extraction can change what a score means even before hardware enters the comparison.\n\n## Reliability and evaluator limits are part of the score\n\nThe [raw tool-evaluation report](https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/tool-eval-bench/tool-eval-bench-8-trials.json) uses tool-eval-bench 2.6.1.dev39+gd3352edf5, 88 hard scenarios and eight trials. The listed trial scores are 88, 88, 86, 87, 88, 88, 88 and 89. The publisher reports a rounded mean 87.8, SD 0.9 and 95 percent interval [87.1, 88.2]; the interval calculation is not independently verified here. The top-level final score is the last trial, not the mean. Pass^8 is 76.1 percent, a stricter repeated-success view than the average score.\n\nThe same ledger's IFEval uses a custom evaluator: strict prompt accuracy is 427/541, or 78.9 percent, and instruction-level accuracy is 710/834, or 85.1 percent. A known suspicious title-format failure motivates checking the evaluator against an official implementation. One suspected false negative does not prove that every discrepancy goes in one direction, so this guide does not certify the score as a mathematical lower bound. Its evaluator remains unverified.\n\nThe [needle report](https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/needle/needle-in-a-haystack-report.md) records 20 successful single-fact probes over four lengths and five depths, through 997,952 actual tokens. The initial 120-second timeout is raised to 3,600 seconds for the same-content rerun. That is finite retrieval evidence, not demonstrated long-context reasoning, concurrency at the full window or a whole-workload peak-memory trace.\n\n![What the historical task scores measure. MMLU uses loglikelihood and no sampling. These are heterogeneous task scores, not a composite.](/assets/batch8-glmflash-scores.webp)\n\n*MMLU uses loglikelihood and no sampling. These are heterogeneous task scores, not a composite.*\n\n## The EXL3 notice changes what can be quoted\n\nThe [October 8 correctness notice](https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/README.md) identifies an unloaded `e_score_correction_bias` in the old cuda-exl3 plugin. The router uses zeros instead of the model's balancing bias and selects wrong experts. The author withdraws the old EXL3 repository's task scores, including its prior MMLU, tool, GSM8K, IFEval and needle panels, until they are rerun. Old speed numbers also do not define corrected routing's speed.\n\nThat notice concerns the EXL3 plugin line. It is not evidence that a separate Marlin NVFP4 implementation has the same defect. Conversely, the separate NVFP4 scores cannot validate the corrected EXL3 model. Keeping those two statements together avoids both discarding unrelated measurements without evidence and filling a corrected stack's missing task scores with a sibling stack's results.\n\nThe [new TP3 main recipe](https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/tracks/tp3-main/README.md) pins target weights to `2a30229e67012798ba9f0cd832bb78abf4c363d5`, base vLLM to `21d93d0d8c0e9627900020382bfce4730e61cab7` and production cuda-exl3 to `448f1d6b8964b96e7d4e299494538bfc240149ef`. It adds a boot gate for the bias. The reported local image ID is retained as an ID, not mislabelled a registry digest. Published patch comments differ from the files in that image, so a byte-identical rebuild is not promised. TP2 has not been ported or tested on this corrected path.\n\n## The rerun is probability health, not a new task leaderboard\n\nThe detailed [KL investigation](https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md) compares the **same quantized checkpoint** to exllamav3 1.5.4, not to BF16. Six author-written texts have 21,072 tokens, 21,066 scored positions and 6,601 assistant positions. The partitioned reference-top-512-plus-remainder estimate is a lower-bound KL measurement; assistant positions form the headline because other regions behave differently.\n\nPre-fix assistant KL is 0.1033; production a4i is 0.00828. The detailed table's perplexity 2.222 belongs to the **bias-fix a4g** arm, against reference 2.224. Production a4i perplexity is unreported in that table, so it remains null even where a short summary groups the numbers together. Neither value gives task accuracy. The prepared text set and full per-position logits are not public, which limits independent reproduction.\n\n![Retractions and reruns belong in the result. Bias-fix perplexity 2.222 belongs to a4g. Production a4i perplexity remains null in the detailed table.](/assets/batch8-glmflash-reruns.webp)\n\n*Bias-fix perplexity 2.222 belongs to a4g. Production a4i perplexity remains null in the detailed table.*\n\nThe [current quality-gate document](https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/quality-gates.md) explicitly says task benchmarks have not been rerun on the corrected stack. The useful publication is therefore a correction with a missing-result ledger: historical NVFP4 tasks retain their own date and harness; withdrawn EXL3 tasks stay withdrawn; corrected EXL3 probability checks retain their reference; future task reruns remain pending. A search or AI summary should return those states alongside any numeric excerpt, rather than selecting the most flattering score from an older table.\n" }, { "id": "A22", "path": "/guides/mimo-pro-eight-spark-scheduler-scores/", "title": "MiMo Pro on Eight Sparks: Scheduling, Rates and Regression Scores", "summary": "Read a13/a14 cold extraction logs, the queued eighth-request fix, full model/base-engine pins, a GSM8K subset and older-build needle checks.", "answer": "The a14 scheduler change raises cold eight-request extraction aggregate rate from 234.57 to 306.66 tokens/s while all 24 answers pass in each arm. This is long output, not a 128K input test: the single-request fixture log has 1498 input tokens. a13 quality scores and older needle runs keep their original build labels.", "published_on": "2026-10-18", "modified_on": "2026-10-18", "source_checked_on": "2026-10-18", "sources": [ { "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "platform": "github", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "README.md", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md", "document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "last_commit_at": "2026-09-24T18:54:01Z", "url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8", "report_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md", "checked_on": "2026-10-18", "metadata_status": "checked", "record_id": "mimopro-tp8-sglang-scheduler" } ], "supplemental_sources": [ { "document_id": "mimopro-versions", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "versions.lock.json", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/versions.lock.json", "document_sha256": "e67a69bc019e7dc5e1a9c5184190901ca147b5f585ac4d1605ca23fde2b1b20c", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-gsm8k", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a13/gsm8k200.json", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/gsm8k200.json", "document_sha256": "4a281c460a7fa3a07b41cf3856faeba767c229a6d43b874ee7ff5d6729865cff", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-a13-long8", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a13/long-extraction-c8.txt", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/long-extraction-c8.txt", "document_sha256": "443ce4520c8472347ac496024483ff46ab1b9a6a3851803d64bf382b66a26f37", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-a14-long8", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a14/long-extraction-c8.txt", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction-c8.txt", "document_sha256": "36b64993a9c32e4b08be0ee600aa743e3fe4c62c917a6ae3ca05f9560c42233a", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-a14-long1", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "results/a14/long-extraction.txt", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction.txt", "document_sha256": "aa85e6233aaf60ab708d6580efada76287b2d560b4cc65f76f72f1cd3f5ab0d3", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-score-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "eval/gsm8k_eval.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/eval/gsm8k_eval.py", "document_sha256": "207a2f4d38a1870a876bd62f8b9070baaf2a07faccbf25ac9ffe194c64b53353", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-speed-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "bench/mimobench.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/mimobench.py", "document_sha256": "d0425c39d1e5ac0f768ea8a451e39a3ad552e8808c2a5c269061faea3bb0b918", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-long1-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "bench/long_extraction_c1.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c1.py", "document_sha256": "3012886b036a75e049296c0d9a02aff20898003c806821fd39e1b8405eb6aabd", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "mimopro-long8-method", "parent_record_id": "mimopro-tp8-sglang-scheduler", "publisher": "rhys101", "repository": "rhys101/MiMo-V2.6-Pro-RL-Spark-8", "revision": "2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10", "document_path": "bench/long_extraction_c8.py", "pinned_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c8.py", "document_sha256": "3c27bb057d58d7afef9fd137bb7a67bb9341a05c85e0f29947e12362b37c685a", "last_commit_at": "2026-09-24T18:54:01Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" } ], "facts": [], "reported_results": [ { "label": "coding C1", "metric": "decode_rate", "value": 91.1, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "metric_denominator": "source per-stream decode" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 28, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L28", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "coding C8 per stream", "metric": "decode_rate", "value": 42.3, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "metric_denominator": "source per-stream decode" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 28, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L28", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "eight categories C8 aggregate", "metric": "aggregate_output_rate", "value": 190.8, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "metric_denominator": "all eight category output / total wall; not mean per-stream multiplied by 8" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 35, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L35", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "a13 cold CSV extraction C8", "metric": "aggregate_output_rate", "value": 234.57, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": "three waves, 8 requests per wave", "cache_state": "unique cache salt; prefix cache flushed before each wave", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "input_length_source_label": "four CSV fixtures, duplicated per wave; C8 input counts not retained", "output_tokens_cap": 4096, "actual_output_tokens_group": 43908, "group_wall_seconds": 187.18, "correct": 24, "total": 24, "seed": 20260922, "metric_denominator": "all completion usage tokens / total synchronized wave wall including prefill" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-a13-long8", "source_document_sha256": "443ce4520c8472347ac496024483ff46ab1b9a6a3851803d64bf382b66a26f37", "source_line": 4, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/long-extraction-c8.txt#L4", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "a14 cold CSV extraction C8", "metric": "aggregate_output_rate", "value": 306.66, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": "sha256:db156b3c950a0ec9c4a7d629f7574f65c1023150bcf2d0755c45d4f2c3cab661", "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a14", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": "three waves, 8 requests per wave", "cache_state": "unique cache salt; prefix cache flushed before each wave", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 8, "input_length_source_label": "four CSV fixtures, duplicated per wave; C8 input counts not retained", "output_tokens_cap": 4096, "actual_output_tokens_group": 43900, "group_wall_seconds": 143.16, "correct": 24, "total": 24, "seed": 20260922, "metric_denominator": "all completion usage tokens / total synchronized wave wall including prefill" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-a14-long8", "source_document_sha256": "36b64993a9c32e4b08be0ee600aa743e3fe4c62c917a6ae3ca05f9560c42233a", "source_line": 4, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction-c8.txt#L4", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "a14 cold CSV extraction C1", "metric": "aggregate_output_rate", "value": 104.6, "unit": "tokens/s", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a14", "temperature": 0, "thinking": false, "input_tokens": 1498, "output_tokens": null, "repetitions": "four tasks x three repeats", "cache_state": "unique salt and cache flush before every request", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "output_tokens_cap": 4096, "actual_output_tokens_group": 21952, "group_wall_seconds": 209.86, "correct": 12, "total": 12, "seed": 20260922, "metric_denominator": "completion usage / summed client wall including prefill" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-a14-long1", "source_document_sha256": "aa85e6233aaf60ab708d6580efada76287b2d560b4cc65f76f72f1cd3f5ab0d3", "source_line": 13, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction.txt#L13", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "a13 GSM8K first 200", "metric": "accuracy", "value": 98, "unit": "percent", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": null, "harness_default_concurrency": 8, "actual_harness_concurrency_reported": false, "benchmark": "first 200 GSM8K test questions in fixed order", "correct": 196, "total": 200, "shots": 0, "output_tokens_cap": 1024, "answer_extraction": "Answer: numeric, else last numeric; abs tolerance <1e-6", "wrong_ids": [ 93, 119, 162, 187 ], "a14_GSM8K_rerun": false, "leaderboard_equivalent": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 83, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L83", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "a13 5K cold prefill", "metric": "prefill_rate", "value": 1880, "unit": "tokens/s", "statistic": "median of three source runs; no SD", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": 3, "cache_state": "unique cold prefix", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "input_length_source_label": "5K", "statistic": "median", "metric_denominator": "TTFT based" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 57, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L57", "source_record_id": "mimopro-tp8-sglang-scheduler" }, { "label": "a13 163K cold prefill", "metric": "prefill_rate", "value": 1499, "unit": "tokens/s", "statistic": "median of three source runs; no SD", "uncertainty": null, "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": 3, "cache_state": "unique cold prefix", "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null, "concurrency": 1, "input_length_source_label": "163K", "statistic": "median", "metric_denominator": "TTFT based" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 58, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L58", "source_record_id": "mimopro-tp8-sglang-scheduler" } ], "conditions": { "model": "MiMo-V2.6-Pro-RL", "model_artifact": "XiaomiMiMo/MiMo-V2.6-Pro-RL", "model_revision": "73875d00b30a89ef8cc353a0b60b0e9f9561952d", "engine": "patched SGLang with Marlin W4A16 MXFP4 experts, DFlash8 adaptive verify and FA4", "base_engine_revision": "06008c170dae333e22caceb51f36224071f9b5b3", "runtime_execution_revision": null, "executed_image_digest": null, "executed_image_id": null, "system": "eight DGX Spark GB10, two switched 200G RoCEv2 subnets, MTU9000", "node_count": 8, "parallelism": "TP8/EP1", "weight_precision": "MXFP4 expert storage; Marlin W4A16; optional FP8 o_proj and draft", "kv_dtype": null, "experiment_date": "2026-09-24", "measured_build": "a13", "temperature": 0, "thinking": false, "input_tokens": null, "output_tokens": null, "repetitions": null, "cache_state": null, "peak_runtime_RAM": null, "context_limit_tokens": null, "drafter_revision": null }, "generation": { "text_draft": "local Spark Qwen, then Codex factual review and rewrite; actual receipt in editorial-batch8 report", "review": "Run AI Here with Codex", "image": "deterministic matplotlib data figures and evidence maps", "local_first": true }, "image": { "url": "/assets/batch8-mimopro-hero.webp", "alt": "MiMo Pro: remove the queued eighth request. 43908/187.18 s vs 43900/143.16 s. Whole-wave wall includes prefill; this is not a 128K test.", "width": 1600, "height": 900, "sha256": "4e9de2c1d12f7fa642819dbacb7ffbb26ec861fc29afaab3eca43a78a1a60a84", "caption": "43908/187.18 s vs 43900/143.16 s. Whole-wave wall includes prefill; this is not a 128K test." }, "case_reports": [ { "id": "a13-eighth-stream-queued", "classification": "source_reported_scheduler_slot_accounting_bug", "initial_build": "a13", "fixed_build": "a14", "initial_C8_aggregate": 234.57, "fixed_C8_aggregate": 306.66, "initial_tokens": 43908, "fixed_tokens": 43900, "initial_wall_seconds": 187.18, "fixed_wall_seconds": 143.16, "initial_correct": 24, "fixed_correct": 24, "total": 24, "initial_mean_per_stream": 39.46, "fixed_mean_per_stream": 38.4, "initial_max_TTFT_seconds": 46.39, "fixed_max_TTFT_seconds": 8.17, "C1_fixture_input_tokens": 1498, "C8_input_counts_retained": false, "universal_long_context_fix": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 49, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L49" } ], "quality_comparisons": [ { "benchmark": "GSM8K first 200, build a13", "correct": 196, "total": 200, "wrong_ids": [ 93, 119, 162, 187 ], "shots": 0, "thinking": false, "a14_rerun": false, "leaderboard_score": false, "full_test": false } ], "memory_observations": [ { "kind": "weight storage / rank fit description, not peak RAM", "weight_files_GB": 535, "weights_per_rank_GB": [ 66, 69 ], "peak_runtime_RAM": null, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 118, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L118" } ], "long_context_cases": [ { "actual_input_tokens": 127803, "builds": [ "a5", "a6" ], "three_facts": true, "current_a13_a14_rerun": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 84, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L84" }, { "actual_input_tokens": 244168, "builds": [ "a5" ], "three_facts": true, "current_a13_a14_rerun": false, "source_document_id": "mimopro-tp8-readme", "source_document_sha256": "97c24b080dd443c8d0c8479f21c09dfb66ffad3d7d4042e1a3acbb399a9c1bef", "source_line": 84, "evidence_url": "https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md#L84" } ], "schema_version": "1.0.0", "record_type": "source_backed_explainer", "evidence_state": "source_reported", "site_reproduced": false, "metrics": null, "performance_ranking_allowed": false, "body_markdown": "# MiMo Pro on Eight Sparks: Scheduling, Rates and Regression Scores\n\n![MiMo Pro: remove the queued eighth request. 43908/187.18 s vs 43900/143.16 s. Whole-wave wall includes prefill; this is not a 128K test.](/assets/batch8-mimopro-hero.webp)\n\n*43908/187.18 s vs 43900/143.16 s. Whole-wave wall includes prefill; this is not a 128K test.*\n\nThe [eight-Spark MiMo-V2.6-Pro-RL deployment](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/README.md) provides more than a fast decode headline. Its most interpretable comparison preserves a scheduler failure, cold-cache request bodies, completion totals, complete-wave wall time and answer checks across builds a13 and a14. That makes it useful for understanding concurrency behavior. It also shows why a workload called “long extraction” should not automatically be filed as a huge-input context test. Run AI Here has not run these model benchmarks or rented additional hardware.\n\n## What the eight-node recipe requires\n\nThe publisher identifies XiaomiMiMo/MiMo-V2.6-Pro-RL, described as a 1.02T-total, 42B-active MoE, at checkpoint revision `73875d00b30a89ef8cc353a0b60b0e9f9561952d`. Eight GB10 nodes use TP8 with EP1, two switched 200G RoCEv2 subnets and MTU 9,000. Each node has a full local weight copy; rank-local weight fit and total disk size are separate from peak runtime unified memory. The source's roughly 66–69 GB weights per rank is not a measured peak.\n\nThe runtime is a patched SGLang with packed MXFP4 experts on Marlin W4A16, FA4 prefill/verify and DFlash8 with adaptive verification width. Optional FP8 output projections and draft components are part of the measured profile. These details mean “MXFP4 deployment” does not imply all arithmetic is 4-bit, and “eight Sparks” alone does not describe the communication topology. A two-node machine or an Ethernet setup without the stated fabric is another recipe.\n\nThe [version lock](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/versions.lock.json) records base SGLang `06008c170dae333e22caceb51f36224071f9b5b3` and the model revision. A base commit does not fully identify a patched executable tree. The a14 local image ID is retained separately from the base-image digest; a portable distribution digest and a13 executed image ID are not reported. Those nulls are more useful than attaching a14's image to older measurements simply because it is the current serving build.\n\n## The short table and the paired extraction use different rates\n\nThe September 24 a13 table reports coding decode at 91.1 tokens/s for C1 and 42.3 per stream at C8. Its all-eight-category C8 aggregate is 190.8 tokens/s. The arithmetic mean across category rates and aggregate tokens divided by group wall time have different denominators. Multiplying a published mean by eight is not a reconstruction of the aggregate. Output length, acceptance behavior and task category also change which rate a user experiences.\n\nThe source warns that individual short requests can move substantially between builds. That is a variability warning, not an estimated confidence interval or proof that every apparent gain is statistically established. The cold structured-extraction comparison is a more inspectable scheduling case because its original logs retain actual output counts and wave timings.\n\n## The eighth request was queued by slot accounting\n\nBuild a13 counts a request finishing its final prefill chunk against a request slot that it already holds. In the affected batch, seven streams run while the eighth waits until another finishes. Build a14 changes that scheduler accounting. The report's explanation is bounded to this path; it is not proof that all queueing in SGLang or all long-context failures have been fixed.\n\n| Cold extraction, three waves of eight | a13 | a14 |\n|---|---:|---:|\n| Correct requests | 24/24 | 24/24 |\n| Completion usage tokens | 43,908 | 43,900 |\n| Total wave wall time, seconds | 187.18 | 143.16 |\n| Aggregate tokens/s, prefill included | 234.57 | 306.66 |\n| Arithmetic mean per-stream tokens/s | 39.46 | 38.40 |\n\nThese values come from the [a13 raw waves](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/long-extraction-c8.txt) and [a14 raw waves](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction-c8.txt). The output totals are close but not identical, and both arms meet the fixture's answer check. The improved aggregate accompanies a lower arithmetic per-stream mean. That is possible because the long queued tail changes the complete-wave denominator, while averaging individual stream rates weights the requests differently.\n\n![Aggregate rate and per-stream mean differ. Do not multiply the arithmetic mean by eight or combine separate denominator definitions.](/assets/batch8-mimopro-denominators.webp)\n\n*Do not multiply the arithmetic mean by eight or combine separate denominator definitions.*\n\nThe worst retained first-token observation falls from 46.39 seconds to 8.17 seconds. This explains the source's rounded 46-to-eight-second description without inventing p95 latency or a distribution of production traffic. Three waves are finite repeated fixtures, not a sustained unattended service test.\n\n## Long extraction is principally a long-output task here\n\nThe [C8 harness](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c8.py) duplicates four CSV-to-JSON tasks per wave, uses temperature zero, seed 20260922, thinking off and a 4,096-token output cap. It supplies a unique cache salt, flushes prefix cache before each wave, consumes completion usage and requires both normal stop and an exact parsed-JSON oracle match. This is stronger evidence than HTTP success alone, while still covering only those fixtures.\n\nThe [C1 raw log](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a14/long-extraction.txt) records **1,498 input tokens** on every request, with about 1,829 output tokens. Twelve requests pass, with 21,952 completion tokens over 209.86 seconds and 104.6 end-to-end tokens/s. The [C1 method](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/bench/long_extraction_c1.py) explicitly obtains the final tuple field from `usage.prompt_tokens`. C8 logs omit prompt counts, so those stay unknown even though they use the related fixtures. Neither table is a 128K-plus concurrent-input experiment.\n\nSeparate a13 cold-prefill medians use source labels 5K, 20K, 81K and 163K, three repeats and a unique prefix. Selected rates are 1,880 tokens/s at nominal 5K and 1,499 at nominal 163K. Exact per-cell input counts are not supplied in the selected summary. These prefill screens should not be mixed with decode-only rates or used as a missing peak-memory trace.\n\n## Keep the score's build, prompting and wrong answers\n\nThe [a13 GSM8K result](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/results/a13/gsm8k200.json) has 196 correct of the first 200 fixed-order test questions: 98.0 percent on that subset. Failed IDs are 93, 119, 162 and 187. The [scoring method](https://github.com/rhys101/MiMo-V2.6-Pro-RL-Spark-8/blob/2f3e5fd804d341ff55a7a7cfdef1d9b782c80c10/eval/gsm8k_eval.py) uses zero-shot brief-working prompts, greedy generation with thinking off, a 1,024-token output cap and a custom numeric extractor with a last-number fallback. Its default concurrency is eight, but the saved result does not establish the executed command's concurrency.\n\n![A regression subset, with its failures. Needle results at 127,803 and 244,168 tokens belong to older a5/a6 builds, not a13/a14.](/assets/batch8-mimopro-score.webp)\n\n*Needle results at 127,803 and 244,168 tokens belong to older a5/a6 builds, not a13/a14.*\n\nThis is a regression check between kernel/backend choices. It is not a full GSM8K leaderboard score, nor comparable to an eight-shot thinking-enabled subset without a matched protocol. There is no a14 GSM8K rerun in the selected evidence. The scheduler fix's 24 correct extraction answers do not fill that missing task rerun.\n\n## Older needle checks do not automatically follow the current image\n\nThree-fact needle results at 127,803 input tokens belong to a5/a6, while 244,168 belongs to a5. Mixed-workload success is also documented on older builds. Current a14 finite smoke checks include text, reasoning split, tool use and a simple image task; audio and video are not validated. These are useful capabilities and gaps, not permission to move an older long-context result onto every subsequent build.\n\n![Eight-node deployment has a real fabric requirement. Image ID, base image digest and the actual patched engine revision are separate provenance fields.](/assets/batch8-mimopro-setup.webp)\n\n*Image ID, base image digest and the actual patched engine revision are separate provenance fields.*\n\nFor deployment planning, the missing next evidence is clear: a14 long-input concurrent tests with actual per-request token counts, synchronized correctness, GPU/unified and host-memory telemetry, and a rerun of the task panel. Retaining the initial scheduling defect and its exact retest makes that next test concrete. A static evidence page can then show what changed, what passed and what remains unmeasured without building an elaborate serving backend or inventing a universal model score.\n" } ] } ] }