# Qwen Long Concurrency: Paired Memory Peaks and OOM Recovery Read two 250K-request memory controls, a 500K single-request peak, a production OOM and invalidated/repeated recovery tests without mixing hardware or cohorts. - Type: Performance report - Evidence: source_reported; site reproduced: false - Model family: Qwen - Model: community NVFP4; exact upstream artifact not identified in the selected report - Original author: nanogenomic - Original project: https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000 - Pinned document: https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md - Source revision: 11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2 - Source commit time (not experiment date): 2026-10-01T04:27:43Z - Checked: 2026-10-18 - Experiment date: 2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported - Runtime topics: SGLang - Hardware topics: nvidia - Reported setup: {"system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "kv_dtype": "FP8", "context_limit_tokens": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported"} - Performance figures: attributed selections with per-point conditions; no cross-source ranking ## Conditions Unknown is null. Recipe options are not measured benchmark settings. ```json { "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "test_profiles": [ { "label": "on-GPU control minimum free VRAM", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" } }, { "label": "on-GPU control non-pool peak over idle", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" } }, { "label": "host KV minimum free VRAM", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" } }, { "label": "host KV non-pool peak over idle", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" } }, { "label": "524288 FP8 old gather isolated transient", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "old gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true } }, { "label": "524288 FP8 bounded gather isolated transient", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "bounded gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true } }, { "label": "500K old gather device peak", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "old gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 } }, { "label": "500K bounded gather device peak", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "bounded gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 } }, { "label": "2x250K host-KV old gather non-pool peak", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "old gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false } }, { "label": "2x250K host-KV bounded gather non-pool peak", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "bounded gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false } }, { "label": "Production OOM failed_allocation", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L639", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" } }, { "label": "Production OOM available_GPU_memory", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L640", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" } }, { "label": "Production OOM process_device_memory", "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L642", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" } } ], "recipe_parameters": null, "deployment_options": [] } ``` ## Reading notes - Selected values are source excerpts, not independent experiment counts. ## Limits - No independent Run AI Here model test. - Read source date, exact execution identity, denominator, memory scope and retries per observation. - No unmatched quality ranking or hardware speed ranking. ## License declaration {"declared_spdx": null, "scope": "Attribution and finite original factual interpretation only; models and dependencies retain their own terms."} ## Editorial answer A bounded gather lowers the isolated transient and the 500K single-request device peak, but the paired two-250K host-KV cell reports the same +1248 MiB non-pool peak on both trees. Production failure, development recovery and the smaller ship-tree rerun have different conditions. By Run AI Here editorial ยท 2026-10-18 Local Spark Qwen draft; Codex source audit and rewrite. No site model benchmark. ## Which memory observations are directly paired? The original report compares cold 2x250K prefill on the same Server Edition 600 W card. Minimum free device memory is 3414 MiB on the on-GPU control versus 6962 MiB with host KV; non-pool peaks over idle are 1090 versus 1248 MiB. Pools differ, so this is an operational configuration comparison. Another same-host-KV old/new gather comparison reports +1248 MiB in both arms, with no measurable peak benefit in that concurrent cell. Sources: https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md, https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/README.md ## Where does bounded gather reduce the peak? The isolated 524288-token FP8 kernel reports 1024 to 32 MiB transient allocation, excluding output, with bit-exact checks in two runs. The in-engine single-request 500000-token sweep reports 97237 to 96093 MiB device used peak at the same 689728-token pool and 94661 MiB idle. These are different measurement scopes and cannot replace the concurrent cell. Host-UVA timing fails to reproduce consistently. Sources: https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md ## What failed in production? The September 30 Max-Q Workstation 300 W lane fails an eager-prefill 560 MiB allocation with 280.94 MiB free and 93.92 GiB process device usage. At failure one request runs, fourteen queue, token pool usage is 0.73 and Mamba usage 0.06. The outage is 12m54s. Monitoring existed but was not consumed by admission. No total host peak or exact failing request input is published. Sources: https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md ## How much recovery was actually retested? The host-KV-off development branch has 45 and 42 recoveries across two injection/real-pressure batteries. The ship tree first has an invalid harness run with zero injections; its corrected rerun has two injections and two recoveries. Decode OOM and driver errors remain fatal. The composite gate lacks NIAH cells in its own bundle even though a separate invocation passes nine needles. Sources: https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md ## Selected source-reported figures Each point retains its own configuration, measurement unit and source. No site reproduction or cross-source ranking. ```json [ { "label": "on-GPU control minimum free VRAM", "metric": "available_GPU_memory", "value": 3414, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 870, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870" }, { "label": "on-GPU control non-pool peak over idle", "metric": "non_pool_memory_over_idle", "value": 1090, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 871, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871" }, { "label": "host KV minimum free VRAM", "metric": "available_GPU_memory", "value": 6962, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 870, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870" }, { "label": "host KV non-pool peak over idle", "metric": "non_pool_memory_over_idle", "value": 1248, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 871, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871" }, { "label": "524288 FP8 old gather isolated transient", "metric": "kernel_transient_memory", "value": 1024.0, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "old gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 954, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954" }, { "label": "524288 FP8 bounded gather isolated transient", "metric": "kernel_transient_memory", "value": 32.0, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "bounded gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 954, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954" }, { "label": "500K old gather device peak", "metric": "peak_device_used_memory", "value": 97237, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "old gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 972, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972" }, { "label": "500K bounded gather device peak", "metric": "peak_device_used_memory", "value": 96093, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "bounded gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 972, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972" }, { "label": "2x250K host-KV old gather non-pool peak", "metric": "non_pool_memory_over_idle", "value": 1248, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "old gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 981, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981" }, { "label": "2x250K host-KV bounded gather non-pool peak", "metric": "non_pool_memory_over_idle", "value": 1248, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "bounded gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 981, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981" }, { "label": "Production OOM failed_allocation", "metric": "failed_allocation", "value": 560.0, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 639, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L639" }, { "label": "Production OOM available_GPU_memory", "metric": "available_GPU_memory", "value": 280.94, "unit": "MiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 640, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L640" }, { "label": "Production OOM process_device_memory", "metric": "process_device_memory", "value": 93.92, "unit": "GiB", "statistic": "source-reported observation; dispersion unreported", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 642, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L642" } ] ``` ## Supplemental source identities ```json { "supplemental_sources": [ { "document_id": "qwen-nanogen-readme", "parent_record_id": "qwen-flash-next-nanogen-memory-recovery", "publisher": "nanogenomic", "repository": "nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000", "revision": "11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2", "document_path": "README.md", "pinned_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/README.md", "document_sha256": "bcfd94276d2bebef681e8a7737444ecec9814d6c73dce9ecf95a0b9aa437a3ab", "last_commit_at": "2026-10-01T04:27:43Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "qwen-nanogen-commits", "parent_record_id": "qwen-flash-next-nanogen-memory-recovery", "publisher": "nanogenomic", "repository": "nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000", "revision": "11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2", "document_path": "COMMIT-MAP.tsv", "pinned_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/COMMIT-MAP.tsv", "document_sha256": "7a6b188d895f6d99fe46ca650832e7d1d8f182aa0b2c59e1442bd7794042fa92", "last_commit_at": "2026-10-01T04:27:43Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" } ] } ``` ## memory observations ```json { "memory_observations": [ { "label": "on-GPU control minimum free VRAM", "value": 3414, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870" }, { "label": "on-GPU control non-pool peak over idle", "value": 1090, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "on-GPU control", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871" }, { "label": "host KV minimum free VRAM", "value": 6962, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "minimum_free_device_memory" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L870" }, { "label": "host KV non-pool peak over idle", "value": 1248, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.15b", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "host KV", "memory_scope": "during cold 2x250K prefill", "metric_kind": "peak_device_nonpool_delta_over_idle" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L871" }, { "label": "524288 FP8 old gather isolated transient", "value": 1024.0, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "old gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954" }, { "label": "524288 FP8 bounded gather isolated transient", "value": 32.0, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": null, "input_tokens": 524288, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 microbenchmark", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": 2, "arm": "bounded gather", "memory_scope": "gather allocation only, excluding output", "bit_exact_all_12_cases_both_runs": true }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L954" }, { "label": "500K old gather device peak", "value": 97237, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "old gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972" }, { "label": "500K bounded gather device peak", "value": 96093, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": 500000, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 in-engine A/B", "cache_state": "same-pool in-engine NIAH", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": "three depths at each of 100000/300000/500000", "arm": "bounded gather", "memory_scope": "in-engine device used peak across three needle depths", "KV_pool_tokens": 689728, "idle_device_MiB": 94661, "needle_passes": 9, "needle_trials": 9 }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L972" }, { "label": "2x250K host-KV old gather non-pool peak", "value": 1248, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "old gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981" }, { "label": "2x250K host-KV bounded gather non-pool peak", "value": 1248, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "one RTX PRO 6000 Blackwell Server Edition, 600 W, PCIe Gen5 x16, driver 580.95.05; 97887 MiB", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 2, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30 to 2026-10-01 source campaign; exact validation-cell timestamps unreported", "host_peak_RAM": null, "source_section": "3.16 concurrent A/B", "cache_state": "cold prefill, two approximately 250K requests", "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "arm": "bounded gather", "memory_scope": "paired concurrent host-KV cold-prefill peak over idle", "measurable_peak_improvement": false }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L981" }, { "label": "Production OOM failed_allocation", "value": 560.0, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L639" }, { "label": "Production OOM available_GPU_memory", "value": 280.94, "unit": "MiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L640" }, { "label": "Production OOM process_device_memory", "value": 93.92, "unit": "GiB", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L642" } ] } ``` ## long context cases ```json { "long_context_cases": [ { "id": "host-KV-two-mains-turn3", "input_length_source_labels": [ "250K", "200K" ], "additional_subagents": 2, "host_KV_pool_tokens": 1310720, "control_pool_tokens": 451840, "host_KV_TTFT_seconds": [ 1.2, 1.0 ], "control_TTFT_seconds": [ 22.9, 41.1 ], "host_KV_cache_hit_rate": 0.9949, "control_cache_hit_rate": 0, "control_requests_completed": true, "host_peak_RAM": null, "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 833, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L833" } ] } ``` ## case reports ```json { "case_reports": [ { "id": "deep-queue-production-OOM", "classification": "source_reported_real_allocator_failure", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 633, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L633", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "community NVFP4; exact upstream artifact not identified in the selected report", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived patched QSA host-KV / bounded-gather branches", "engine_base_short": "6fa3fe69e2", "system": "RTX PRO 6000 Blackwell Max-Q Workstation, 300 W, PCIe Gen4 x16, driver 610.43.02", "node_count": 1, "parallelism": "TP1", "weight_precision": "NVFP4", "kv_dtype": "FP8", "PLE": "host pinned RAM", "concurrency": 1, "input_tokens": null, "output_tokens": null, "temperature": null, "thinking": null, "experiment_date": "2026-09-30", "host_peak_RAM": null, "source_section": "3.14 production failure", "cache_state": null, "control_pool_tokens": 451840, "host_KV_pool_tokens": 1310720, "controlled_pool_sizes_equal": false, "repetitions": null, "queued_requests": 14, "pending_tokens": 1485849, "full_token_usage": 0.73, "mamba_usage": 0.06, "KV_pool_tokens": 522880, "memory_fraction_static": 0.98, "other_process_device_MiB": 676, "exact_failed_request_input_tokens": null, "memory_scope": "allocator failure snapshot; not a sampled campaign peak" }, "downtime_seconds": 774, "error": "torch.OutOfMemoryError in eager prefill; scheduler SIGQUIT and process-tree termination", "monitoring_alert_before_crash_seconds": 53, "admission_consumed_alert": false }, { "id": "recovery-branch-versus-ship", "classification": "source_reported_fault_injection_and_real_pressure", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 999, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L999", "branch": "oom-retract, host KV off", "branch_pool_tokens": [ 451712, 451840 ], "runs": [ { "forced": 39, "real": 6, "recoveries": 45 }, { "forced": 39, "real": 3, "recoveries": 42 } ], "engine_alive": true, "scheduler_suite": { "initial_harness_errors": 9, "initial_error": "runner IndexError", "rerun_passed": 100 }, "ship_tree": { "initial_valid": false, "initial_error": "BatchEncoding not JSON serializable", "initial_injects": 0, "initial_recoveries": 0, "initial_verdict": false, "rerun_injections": 2, "rerun_recoveries": 2, "rerun_all_needles_correct": true, "engine_alive": true, "host_KV_pool_tokens": 1310720 }, "recovery_predicates": [ "overlap enabled", "EXTEND", "TP1", "PP1", "no MLP sync", "no disaggregation", "HiSparse off", "PDMux off", "no diffusion config", "device-wide drain completes" ], "max_retractions": 3, "exhaustion_status": 503, "next_request_status": 200, "decode_OOM_recovered": false, "driver_error_recovered": false, "peer_outputs_bit_identical": false }, { "id": "host-UVA-regression", "classification": "unresolved_nonreproducing_microbenchmark", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 992, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L992", "first_new_ms": [ 341, 376 ], "later_new_ms": [ 4.99, 5.68 ], "later_new_peak_MiB": [ 159, 189 ], "first_touch_explanation_proven": false, "kernel_time_improvement_claim": false }, { "id": "composite-gate-missing-same-bundle-NIAH", "classification": "harness_composition_failure", "source_document_id": "qwen-nanogen-bench", "source_document_sha256": "7015f023635f4d1503e7e6df2ab373ebf87aafbce2cdd127c29f6ff2a8b77076", "source_line": 873, "evidence_url": "https://github.com/nanogenomic/qwen-3-8-flash-next-RTX-PRO-6000/blob/11a09ff7fbf2cdb60dbccb2e345fdfb9aa43f2f2/BENCHMARKS.md#L873", "decode_path_gate": true, "standalone_needle_correct": 9, "standalone_needle_total": 9, "same_bundle_NIAH_cells": 0, "long_context_gate": false, "overall_gate": false, "model_quality_failure_established": false } ] } ```