# Qwen Long Context: Concurrency, Memory and Failure Evidence Read 128K and 490K Qwen batches, a four-GPU 128K matrix, sampled peak VRAM and a Mamba crash with original versions and measurement limits. - Type: Performance report - Evidence: source_reported; site reproduced: false - Model family: Qwen - Model: RadixArk/Qwen3.8-Flash-Next-NVFP4 - Original author: jpezzulli - Original project: https://github.com/jpezzulli/sglang-rtxpro6000 - Pinned document: https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md - Source revision: ca9be12f9be920e916d02954a41702a981b18823 - Source commit time (not experiment date): 2026-10-17T16:41:23Z - Checked: 2026-10-18 - Experiment date: 2026-09-11 - Runtime topics: SGLang, vLLM - Hardware topics: nvidia - Reported setup: {"system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "kv_dtype": "FP8", "context_limit_tokens": 524288, "experiment_date": "2026-09-11"} - Performance figures: attributed selections with per-point conditions; no cross-source ranking ## Conditions Unknown is null. Recipe options are not measured benchmark settings. ```json { "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "test_profiles": [ { "label": "128K C4 option off", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L231", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K four-request three-key retrieval", "input_length_source_label": "128K", "concurrency": 4, "precision_option": "option off", "actual_output_tokens_group": 15725, "group_wall_seconds": 42.833, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": null, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false } }, { "label": "128K C4 online FP8", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L231", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K four-request three-key retrieval", "input_length_source_label": "128K", "concurrency": 4, "precision_option": "online FP8", "actual_output_tokens_group": 15898, "group_wall_seconds": 37.67, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": 30.152, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false } }, { "label": "490K C4 option off", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L232", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K four-request three-key retrieval", "input_length_source_label": "490K", "concurrency": 4, "precision_option": "option off", "actual_output_tokens_group": 15455, "group_wall_seconds": 65.3, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": null, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false } }, { "label": "490K C4 online FP8", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L232", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K four-request three-key retrieval", "input_length_source_label": "490K", "concurrency": 4, "precision_option": "online FP8", "actual_output_tokens_group": 15127, "group_wall_seconds": 45.977, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": 24.85, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false } }, { "label": "128K online FP8 C1 decode_rate", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L229", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "128K", "metric_denominator": "client generation after first token, excluding TTFT" } }, { "label": "128K online FP8 C1 ttft", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L229", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "128K", "metric_denominator": "client generation after first token, excluding TTFT" } }, { "label": "490K online FP8 C1 decode_rate", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L230", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "490K", "metric_denominator": "client generation after first token, excluding TTFT" } }, { "label": "490K online FP8 C1 ttft", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L230", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "490K", "metric_denominator": "client generation after first token, excluding TTFT" } }, { "label": "Post-graph free VRAM", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L243", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "concurrency": null, "memory_kind": "post_graph_available_not_peak_used", "memory_cohort": "startup", "paired_long_text_batch_peak": false } }, { "label": "Media-window minimum free VRAM", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L263", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "concurrency": null, "memory_kind": "sampled_minimum_free_not_per_long_text_batch_peak", "memory_cohort": "ten selected GPU-media checks; ordinary/large JPEG, concurrent images and video history", "paired_long_text_batch_peak": false } }, { "label": "3.0 cold 64K prefill_rate", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L44", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 63921, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "native mean input rate" } }, { "label": "3.0 cold 64K ttft", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L44", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 63921, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "client first-token latency" } }, { "label": "3.0 cold 490K prefill_rate", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L45", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 489915, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "native mean input rate" } }, { "label": "3.0 cold 490K ttft", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L45", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 489915, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "client first-token latency" } }, { "label": "3.0 short C1 native mean", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L26", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": 0.8, "top_p": 0.95, "thinking": "xhigh", "repetitions": 3, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 128, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "regular warmed screens", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "top_k": 40, "min_p": 0.05, "metric_denominator": "elapsed-weighted complete active native log intervals; excludes idle/prefill crossing and partial tail; aggregate if C4" } }, { "label": "3.0 short C4 native mean", "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L27", "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": 0.8, "top_p": 0.95, "thinking": "xhigh", "repetitions": 3, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 128, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "regular warmed screens", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 4, "top_k": 40, "min_p": 0.05, "metric_denominator": "elapsed-weighted complete active native log intervals; excludes idle/prefill crossing and partial tail; aggregate if C4" } } ], "recipe_parameters": null, "deployment_options": [] } ``` ## Reading notes - A source document and its selected result points are not independent experiment counts. ## Limits - No independent Run AI Here model test. - Executed identity, timing definition, cache state, repetition unit and missingness must be read per result. - No cross-hardware or unmatched model score ranking. ## License declaration {"declared_spdx": null, "scope": "Attribution and finite factual interpretation only; models and dependencies retain their own terms."} ## Editorial answer Four submitted requests do not guarantee four running requests. The sources contain completed long-context batches, but their cache states, timing denominators and memory windows differ. Keep the four-GPU campaign peak separate from per-cell latency, and retain the original scheduler failure. By Run AI Here editorial ยท 2026-10-18 Local Spark Qwen writing draft; Codex source audit and rewrite. No site model benchmark. ## Which long-context batches were measured? Penny Royal v2.5.0 tests one RTX PRO 6000 at TP1 with RadixArk NVFP4, RAM PLE, FP8 KV, FR-Spec and an 824384-token pool. Selected 128K and 490K C4 results divide all output by synchronized group wall time. Each cell is one batch, with stop or a 4096-token cap and differing completion totals. The online-FP8 arm records actual four-way overlap; exact prefix sharing and full checkpoint revision remain unreported here. Sources: https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md ## What does the newer release add? October 10 Penny Royal 3.0 has separate short regular screens and actual 63921/489915-token cold prompts. It uses another checkpoint, an adaptive policy and a larger token pool. The regular rate is an elapsed-weighted native active-interval mean, not whole task wall time. Public accepted source and local diagnostic checkout are separately identified. These cells do not replace the older long C4 experiment. Sources: https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md, https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/release-3.0.json ## How are memory and sustained-load evidence bounded? The external four-GPU report supplies a sampled summed used-VRAM peak for its entire matrix, not a per-cell peak or host peak. Penny Royal post-graph free VRAM and minimum free VRAM in a GPU-media window are different observations. A separate credited field report submits four approximately 240K requests over ten rounds, with queuing and 57 percent peak pool occupancy; HTTP 200 alone is not retrieval correctness. Sources: https://www.cstech.dev/articles/qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell, https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md, https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/COMMUNITY-RESULTS.md ## Which failure remains useful? A first-person SGLang incident has approximately 140K cold input, HiCache write-through, 51 Mamba slots and only two running requests when Mamba usage reaches 1.00. The scheduler asserts while full token usage is 0.46. PR37619 is still open at this check; its CPU tests and proposed fallback do not establish that the GPU reproducer or all affected workloads are fixed. Sources: https://github.com/sgl-project/sglang/pull/37619#issuecomment-5606493113 ## Selected source-reported figures Each point retains its own configuration, measurement unit and source. No site reproduction or cross-source ranking. ```json [ { "label": "128K C4 option off", "metric": "aggregate_output_rate", "value": 367.127, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K four-request three-key retrieval", "input_length_source_label": "128K", "concurrency": 4, "precision_option": "option off", "actual_output_tokens_group": 15725, "group_wall_seconds": 42.833, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": null, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 231, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L231" }, { "label": "128K C4 online FP8", "metric": "aggregate_output_rate", "value": 422.037, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K four-request three-key retrieval", "input_length_source_label": "128K", "concurrency": 4, "precision_option": "online FP8", "actual_output_tokens_group": 15898, "group_wall_seconds": 37.67, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": 30.152, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 231, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L231" }, { "label": "490K C4 option off", "metric": "aggregate_output_rate", "value": 236.678, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K four-request three-key retrieval", "input_length_source_label": "490K", "concurrency": 4, "precision_option": "option off", "actual_output_tokens_group": 15455, "group_wall_seconds": 65.3, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": null, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 232, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L232" }, { "label": "490K C4 online FP8", "metric": "aggregate_output_rate", "value": 329.01, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K four-request three-key retrieval", "input_length_source_label": "490K", "concurrency": 4, "precision_option": "online FP8", "actual_output_tokens_group": 15127, "group_wall_seconds": 45.977, "requested_output_cap_per_request": 4096, "finish_reasons": [ "stop", "length" ], "all_four_overlap_seconds": 24.85, "metric_denominator": "all group output tokens / synchronized group makespan including TTFT", "all_responses_three_keys_exact": true, "all_four_full_distinct_prefixes_proven": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 232, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L232" }, { "label": "128K online FP8 C1 decode_rate", "metric": "decode_rate", "value": 195.634, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "128K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 229, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L229" }, { "label": "128K online FP8 C1 ttft", "metric": "ttft", "value": 12.906, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "128K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "128K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 229, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L229" }, { "label": "490K online FP8 C1 decode_rate", "metric": "decode_rate", "value": 172.644, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "490K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 230, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L230" }, { "label": "490K online FP8 C1 ttft", "metric": "ttft", "value": 63.296, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "workload": "490K C1 three-key retrieval", "concurrency": 1, "precision_option": "online FP8", "input_length_source_label": "490K", "metric_denominator": "client generation after first token, excluding TTFT" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 230, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L230" }, { "label": "Post-graph free VRAM", "metric": "available_GPU_memory", "value": 7.52, "unit": "GiB", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "concurrency": null, "memory_kind": "post_graph_available_not_peak_used", "memory_cohort": "startup", "paired_long_text_batch_peak": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 243, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L243" }, { "label": "Media-window minimum free VRAM", "metric": "available_GPU_memory", "value": 1897, "unit": "MiB", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "RadixArk/Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 824384, "mamba_slots": 24, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-09-11", "recipe_release": "v2.5.0", "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": "post-warmup context tests; exact C4 cache sharing unreported", "concurrency": null, "memory_kind": "sampled_minimum_free_not_per_long_text_batch_peak", "memory_cohort": "ten selected GPU-media checks; ordinary/large JPEG, concurrent images and video history", "paired_long_text_batch_peak": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 263, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L263" }, { "label": "3.0 cold 64K prefill_rate", "metric": "prefill_rate", "value": 17235.9, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 63921, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "native mean input rate" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 44, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L44" }, { "label": "3.0 cold 64K ttft", "metric": "ttft", "value": 4.71, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 63921, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "client first-token latency" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 44, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L44" }, { "label": "3.0 cold 490K prefill_rate", "metric": "prefill_rate", "value": 8839.75, "unit": "tokens/s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 489915, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "native mean input rate" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 45, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L45" }, { "label": "3.0 cold 490K ttft", "metric": "ttft", "value": 60.37, "unit": "s", "statistic": "one reported observation; no dispersion", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": null, "top_p": null, "thinking": null, "repetitions": 1, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 489915, "output_tokens": null, "actual_output_tokens": null, "cache_state": "100% uncached prompt", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "metric_denominator": "client first-token latency" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 45, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L45" }, { "label": "3.0 short C1 native mean", "metric": "native_decode_rate", "value": 218.63, "unit": "tokens/s", "statistic": "native mean; three regular repeats, no SD", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": 0.8, "top_p": 0.95, "thinking": "xhigh", "repetitions": 3, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 128, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "regular warmed screens", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 1, "top_k": 40, "min_p": 0.05, "metric_denominator": "elapsed-weighted complete active native log intervals; excludes idle/prefill crossing and partial tail; aggregate if C4" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 26, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L26" }, { "label": "3.0 short C4 native mean", "metric": "native_decode_rate", "value": 489.35, "unit": "tokens/s", "statistic": "native mean; three regular repeats, no SD", "uncertainty": null, "conditions": { "model": "Qwen3.8-Flash-Next", "model_artifact": "OrcaRouter ModelOpt NVFP4 checkpoint; exact revision unreported here", "model_revision": null, "runtime_execution_revision": "90b2b412e2bfd06f0ef3be1b41fbfbdee45bc14f", "executed_image_digest": null, "engine": "SGLang-derived Penny Royal", "system": "one RTX PRO 6000 Blackwell SM120, 96 GB; TP1", "node_count": 1, "parallelism": "TP1", "weight_precision": "ModelOpt NVFP4 experts; optional online FP8 projections", "kv_dtype": "FP8", "PLE": "host RAM", "context_limit_tokens": 524288, "KV_pool_tokens": 1067072, "mamba_slots": 36, "HiCache_NIXL_GiB": 32, "FR_Spec": true, "temperature": 0.8, "top_p": 0.95, "thinking": "xhigh", "repetitions": 3, "experiment_date": "2026-10-10", "recipe_release": "3.0", "input_tokens": 128, "output_tokens": 1024, "actual_output_tokens": 1024, "cache_state": "regular warmed screens", "accepted_public_core": "b48b70f3f80494d78a778fd4d26958ede343075f", "local_runtime_has_private_diagnostics": true, "media_preprocessing": "A4000 sidecar; not an inference TP rank", "concurrency": 4, "top_k": 40, "min_p": 0.05, "metric_denominator": "elapsed-weighted complete active native log intervals; excludes idle/prefill crossing and partial tail; aggregate if C4" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "penny-results", "source_document_sha256": "51376fac57c0f5a277ada7d2a9359d365a7f861b424544c98bd49959b2b18e8d", "source_line": 27, "evidence_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md#L27" } ] ``` ## Supplemental source identities ```json { "supplemental_sources": [ { "document_id": "penny-readme", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "README.md", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/README.md", "document_sha256": "7645e7297eb1326393e8e4106fb73334980dfe44540b45e51655bc1dadd295ad", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "penny-community-results", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "COMMUNITY-RESULTS.md", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/COMMUNITY-RESULTS.md", "document_sha256": "2bbe9428f0701dc597dd0417193c12e72157dff2aca3ee6481b3104755a1301c", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "penny-release-3.0", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "release-3.0.json", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/release-3.0.json", "document_sha256": "1799eca8496bf143fc48bbdc83a4f6ccc34b8773c43083b47e29721b1ceb3dcf", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "penny-release-3.0.2", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "jpezzulli", "repository": "jpezzulli/sglang-rtxpro6000", "revision": "ca9be12f9be920e916d02954a41702a981b18823", "document_path": "release-3.0.2.json", "pinned_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/release-3.0.2.json", "document_sha256": "4843d621fa7f735a4c95d6bc79c9582d465cfaecef6b473689158d8c190920e2", "last_commit_at": "2026-10-17T16:41:23Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "qwen-cstech-4gpu", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "CS Technology Services", "repository": null, "revision": "snapshot-2026-10-18", "document_path": "qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell", "pinned_url": "https://www.cstech.dev/articles/qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell", "document_sha256": "b81878ba45f2f6b7a6de06b0bf930bd1dfa3cb497202ed3fcd368efe66065930", "last_commit_at": "2026-08-26", "checked_on": "2026-10-18", "public_URL_is_immutable": false, "revision_kind": "Mutable original report; retrieval snapshot date and SHA256, no Git revision" }, { "document_id": "qwen-mamba-incident", "parent_record_id": "qwen38-flash-next-rtxpro-long-concurrency", "publisher": "leehom0123", "repository": "sgl-project/sglang", "revision": "comment-5606493113", "document_path": "/issues/37619/comments/5606493113", "pinned_url": "https://github.com/sgl-project/sglang/pull/37619#issuecomment-5606493113", "document_sha256": "d36cc21fa7b9577474ba20c9fe26becbb3c61bab9b9c4a5c421447b9d905c3f4", "last_commit_at": "2026-09-09T18:05:35Z", "checked_on": "2026-10-18", "public_URL_is_immutable": false, "revision_kind": "GitHub issue comment ID; mutable body with saved SHA256" } ] } ``` ## memory observations ```json { "memory_observations": [ { "kind": "sampled_peak_summed_GPU_used_over_entire_matrix", "value": 374976, "unit": "MiB", "sample_count": 501, "campaign_seconds": 1206, "paired_per_cell_peak": null, "host_peak_RAM": null }, { "kind": "v2.5.0_GPU_media_window_minimum_free", "value": 1897, "unit": "MiB", "long_text_C4_peak": null, "source_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/RESULTS.md" } ] } ``` ## long context cases ```json { "long_context_cases": [ { "classification": "source_reported_four_submitted_stress", "recipe_release": "v2.5.0", "contributor": "BESTIA / Prometeus field report; separate contributor from runtime author", "source_url": "https://www.reddit.com/r/BlackwellPerformance/comments/1weaxdz/comment/p9i76fb/", "source_summary_url": "https://github.com/jpezzulli/sglang-rtxpro6000/blob/ca9be12f9be920e916d02954a41702a981b18823/COMMUNITY-RESULTS.md", "context_limit_tokens": 524288, "KV_pool_tokens": 1114304, "input_tokens": null, "input_length_source_label": "approximately 240K", "submitted_concurrency": 4, "simultaneous_running_concurrency": null, "rounds": 10, "total_requests": 40, "output_tokens_reported": 8192, "round_seconds_range": [ 93, 107 ], "HTTP_200": 40, "OOMs_reported": 0, "retractions_reported": 0, "aborts_reported": 0, "peak_pool_occupancy_percent": 57, "peak_memory_bytes": null, "queuing_observed": true, "retrieval_correctness": null } ] } ``` ## case reports ```json { "case_reports": [ { "id": "qwen-cstech-128k-matrix", "classification": "source_reported_duration_matrix", "source_url": "https://www.cstech.dev/articles/qwen3-8-flash-next-fp8-on-4x-rtx-pro-6000-blackwell", "source_snapshot_sha256": "b81878ba45f2f6b7a6de06b0bf930bd1dfa3cb497202ed3fcd368efe66065930", "public_URL_is_immutable": false, "source_published_on": "2026-08-26", "model": "Qwen/Qwen3.8-Flash-Next-FP8", "model_revision": null, "engine": "vLLM 0.1.dev20073+g8e685d198", "executed_image_digest": null, "runtime_execution_revision": null, "harness": "llm-inference-bench v0.4.29", "system": "4 RTX PRO 6000 Blackwell Max-Q, 300 W/card, Threadripper, PCIe Gen4, no NVLink", "parallelism": "TP4", "context_limit_tokens": 262144, "KV_pool_tokens": 2662752, "configured_max_requests": 256, "draft_tokens": 3, "temperature": null, "top_p": null, "thinking": null, "input_length_source_label": "128k nominal matrix context; exact per-cell prompt token counts unreported", "duration_seconds_per_cell": 30, "output_tokens_cap": 8192, "ignore_eos": true, "effective_concurrency_validated_by_source": true, "cache_state": "duration-mode integrated scout warms shared prefix; matrix-start TTFT includes cache hit and queueing", "cells": [ { "concurrency": 1, "aggregate_output_tokens_per_second": 128.5, "TTFT_seconds": 1.41 }, { "concurrency": 8, "aggregate_output_tokens_per_second": 505.0, "TTFT_seconds": 2.02 }, { "concurrency": 16, "aggregate_output_tokens_per_second": 729.9, "TTFT_seconds": 3.42 } ], "omitted_cell": { "concurrency": 32, "status": "not_run_capacity", "aggregate_output_tokens_per_second": null, "observed_runtime_crash": false }, "memory": { "kind": "sampled_peak_summed_GPU_used_over_entire_matrix", "value": 374976, "unit": "MiB", "sample_count": 501, "campaign_seconds": 1206, "paired_per_cell_peak": null, "host_peak_RAM": null }, "cold_C1": { "actual_input_tokens": 128021, "TTFT_seconds": 24.25, "prefill_tokens_per_second": 5279 }, "per_user_table_is_derived_aggregate_divided_by_streams": true }, { "id": "qwen-mamba-long-prefill-crash", "classification": "source_reported_scheduler_assertion", "source_url": "https://github.com/sgl-project/sglang/pull/37619#issuecomment-5606493113", "source_snapshot_sha256": "d36cc21fa7b9577474ba20c9fe26becbb3c61bab9b9c4a5c421447b9d905c3f4", "source_updated_at": "2026-09-09T18:05:35Z", "public_URL_is_immutable": false, "model": "Qwen3.8-Flash-Next-NVFP4", "model_revision": null, "engine": "SGLang 0.0.0.dev1+g4ccff141d", "runtime_execution_revision": null, "executed_image_digest": null, "weight_precision": "modelopt_mixed", "kv_dtype": "fp8_e4m3", "context_length_source_label": "262k", "actual_prompt_tokens": null, "prompt_length_source_label": "approximately 140k", "chunked_prefill_size": 4096, "mamba_track_interval": 64, "mamba_slots": 51, "configured_max_requests": 12, "observed_running_requests": 2, "observed_queued_requests_reporter": 1, "queued_requests_at_cited_log_line": 0, "mamba_usage_before": 0.18, "mamba_usage_at_failure": 1.0, "full_token_usage_at_failure": 0.46, "HiCache": true, "hicache_ratio": 2.0, "write_policy": "write_through", "speculation": false, "error": "AssertionError: Can not alloc mamba cache", "outcome": "scheduler child exited; container auto-restarted, service outage during reload", "fix_reference": "https://github.com/sgl-project/sglang/pull/37619", "fix_PR_state": "open", "fix_PR_merged": false, "fix_PR_head_revision": "cd530ac217ac0a67b913f7083679c0890670d228", "author_GPU_reproducer_rerun": false, "universal_fix_verified": false, "peak_RAM": null } ] } ```