{
  "id": "qwen38-chisel-dflash-replay",
  "title": "Qwen3.8-27B: Chisel replay on one DGX Spark",
  "kind": "benchmark",
  "source_type": "community_operator_report",
  "summary": "A pinned single-Spark NVFP4/DFlash2 recipe with separate speed cohorts, task-level BF16 comparisons and negative non-inferiority flags.",
  "runtime": "vLLM",
  "runtime_topics": [
    "vLLM"
  ],
  "hardware_topics": [
    "spark",
    "nvidia"
  ],
  "platform_topics": [
    "linux"
  ],
  "model_label": "Qwen3.8-27B",
  "model_family": "Qwen",
  "model_id": "unsloth/Qwen3.8-27B-NVFP4",
  "model_release_date": null,
  "model_artifact": {
    "id": "unsloth/Qwen3.8-27B-NVFP4",
    "revision": null
  },
  "evidence_state": "source_reported",
  "site_reproduced": false,
  "record_type": "curated_reference",
  "performance_ranking_allowed": false,
  "metrics": null,
  "metrics_status": "site_metrics_unmeasured; attributed_source_figures_in_editorial.reported_results",
  "conditions": {
    "model_revision": null,
    "weight_hash": null,
    "runtime_execution_revision": null,
    "input_tokens": null,
    "output_tokens": null,
    "concurrency": null,
    "cache_state": null,
    "quantization": null,
    "speculative_decoding": null,
    "temperature": null,
    "top_p": null
  },
  "reported_setup": {
    "system": "1 x DGX Spark GB10; 128 GB unified memory; 2000 MHz cap in candidate timing cohort",
    "node_count": 1,
    "parallelism": "TP1",
    "kv_dtype": null,
    "context_limit_tokens": null,
    "experiment_date": "2026-10-15 source cohort; BF16 reference filed 2026-10-09"
  },
  "test_profiles": [
    {
      "label": "Chisel replay mixed-prompt median decode latency",
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L9",
      "conditions": {
        "model": "Qwen3.8-27B",
        "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
        "model_revision": null,
        "reported_model_revision_prefix": "f0b7c9e7",
        "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
        "node_count": 1,
        "driver": "580.178.04",
        "GPU_clock_cap_MHz": 2000,
        "engine": "vLLM",
        "executed_engine_version": "0.31.1rc1.dev380",
        "runtime_execution_revision": null,
        "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
        "executed_local_image_digest": null,
        "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
        "plugin": "chisel-passes 0.1.0",
        "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
        "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
        "draft_tokens": 10,
        "GDN_state_dtype": "bfloat16",
        "KV_dtype": null,
        "configured_context_limit_tokens": 262144,
        "configured_max_requests": 4,
        "GPU_memory_utilization": 0.6,
        "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
        "thinking": true,
        "reasoning_effort": "medium",
        "greedy": true,
        "temperature": 0,
        "top_p": null,
        "concurrency": 1,
        "input_tokens": null,
        "max_output_tokens": 256,
        "output_tokens": null,
        "prompt_count": 25,
        "prompt_mix": {
          "chat": 9,
          "code": 9,
          "JSON_schema": 6,
          "long_story": 1
        },
        "repeats_per_session": 3,
        "fresh_start_sessions": 2,
        "measurement_denominator": "(last streamed token time - first) / (output tokens - 1); median latency, not whole-request served rate"
      }
    },
    {
      "label": "Chisel replay mixed-prompt median TTFT",
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L10",
      "conditions": {
        "model": "Qwen3.8-27B",
        "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
        "model_revision": null,
        "reported_model_revision_prefix": "f0b7c9e7",
        "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
        "node_count": 1,
        "driver": "580.178.04",
        "GPU_clock_cap_MHz": 2000,
        "engine": "vLLM",
        "executed_engine_version": "0.31.1rc1.dev380",
        "runtime_execution_revision": null,
        "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
        "executed_local_image_digest": null,
        "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
        "plugin": "chisel-passes 0.1.0",
        "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
        "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
        "draft_tokens": 10,
        "GDN_state_dtype": "bfloat16",
        "KV_dtype": null,
        "configured_context_limit_tokens": 262144,
        "configured_max_requests": 4,
        "GPU_memory_utilization": 0.6,
        "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
        "thinking": true,
        "reasoning_effort": "medium",
        "greedy": true,
        "temperature": 0,
        "top_p": null,
        "concurrency": 1,
        "input_tokens": null,
        "max_output_tokens": 256,
        "output_tokens": null,
        "prompt_count": 25,
        "prompt_mix": {
          "chat": 9,
          "code": 9,
          "JSON_schema": 6,
          "long_story": 1
        },
        "repeats_per_session": 3,
        "fresh_start_sessions": 2,
        "measurement_denominator": "request start to first streamed token"
      }
    },
    {
      "label": "Local llama-benchy 16K C1",
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11",
      "conditions": {
        "model": "Qwen3.8-27B",
        "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
        "model_revision": null,
        "reported_model_revision_prefix": "f0b7c9e7",
        "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
        "node_count": 1,
        "driver": "580.178.04",
        "GPU_clock_cap_MHz": 2000,
        "engine": "vLLM",
        "executed_engine_version": "0.31.1rc1.dev380",
        "runtime_execution_revision": null,
        "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
        "executed_local_image_digest": null,
        "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
        "plugin": "chisel-passes 0.1.0",
        "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
        "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
        "draft_tokens": 10,
        "GDN_state_dtype": "bfloat16",
        "KV_dtype": null,
        "configured_context_limit_tokens": 262144,
        "configured_max_requests": 4,
        "GPU_memory_utilization": 0.6,
        "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
        "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores",
        "concurrency": 1,
        "input_tokens": 2048,
        "retained_context_tokens": 16384,
        "output_tokens": 128,
        "prefix_cache": true,
        "repeats_per_session": 5,
        "fresh_start_sessions": 2,
        "thinking": null,
        "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2"
      }
    },
    {
      "label": "Local llama-benchy 16K C2",
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11",
      "conditions": {
        "model": "Qwen3.8-27B",
        "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
        "model_revision": null,
        "reported_model_revision_prefix": "f0b7c9e7",
        "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
        "node_count": 1,
        "driver": "580.178.04",
        "GPU_clock_cap_MHz": 2000,
        "engine": "vLLM",
        "executed_engine_version": "0.31.1rc1.dev380",
        "runtime_execution_revision": null,
        "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
        "executed_local_image_digest": null,
        "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
        "plugin": "chisel-passes 0.1.0",
        "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
        "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
        "draft_tokens": 10,
        "GDN_state_dtype": "bfloat16",
        "KV_dtype": null,
        "configured_context_limit_tokens": 262144,
        "configured_max_requests": 4,
        "GPU_memory_utilization": 0.6,
        "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
        "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores",
        "concurrency": 2,
        "input_tokens": 2048,
        "retained_context_tokens": 16384,
        "output_tokens": 128,
        "prefix_cache": true,
        "repeats_per_session": 5,
        "fresh_start_sessions": 2,
        "thinking": null,
        "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2"
      }
    }
  ],
  "deployment_options": [],
  "recipe_parameters": null,
  "measurement_scope": "inference_speed_and_task_checks",
  "notes": [
    "The recipe is a concrete deployment candidate, but speed and quality use different request shapes and thinking modes. The source custom quality bar passes while its non-inferiority flags are false; preserve both and leave the executed local image digest unknown."
  ],
  "limitations": [
    "Source-reported only; no independent site model runs.",
    "Document revision identifies the report, not the executed runtime or weight revision.",
    "Complete measured configurations and missing fields must accompany extracted values."
  ],
  "comparison_status": "not_comparable",
  "editorial_status": "source_derived_detailed_article",
  "public_benchmark_snapshot": false,
  "source": {
    "publisher": "yuanlin2004",
    "url": "https://github.com/yuanlin2004/chisel-spark-recipes",
    "repository": "yuanlin2004/chisel-spark-recipes",
    "platform": "github",
    "revision": "84feafddff1e834bd16b849c827eed11676e5934",
    "document_path": "recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md",
    "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md",
    "report_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md",
    "document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321",
    "last_commit_at": "2026-10-17T23:01:04Z",
    "checked_on": "2026-10-18",
    "metadata_status": "checked"
  },
  "supplemental_sources": [
    {
      "document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.yaml",
      "parent_record_id": "qwen38-chisel-dflash-replay",
      "publisher": "yuanlin2004",
      "repository": "yuanlin2004/chisel-spark-recipes",
      "revision": "84feafddff1e834bd16b849c827eed11676e5934",
      "document_path": "recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.yaml",
      "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.yaml",
      "document_sha256": "028f2badf76407546be67b8bd93e2e81aae8ad5d00076e20b8bd8835021a1ede",
      "checked_on": "2026-10-18"
    },
    {
      "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-ENVIRONMENT.md",
      "parent_record_id": "qwen38-chisel-dflash-replay",
      "publisher": "yuanlin2004",
      "repository": "yuanlin2004/chisel-spark-recipes",
      "revision": "84feafddff1e834bd16b849c827eed11676e5934",
      "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/ENVIRONMENT.md",
      "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/ENVIRONMENT.md",
      "document_sha256": "349c8de9b51a1df3ebba677db79d4e62c9a652a6dc225965f7515b80583d471a",
      "checked_on": "2026-10-18"
    },
    {
      "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-METHOD.md",
      "parent_record_id": "qwen38-chisel-dflash-replay",
      "publisher": "yuanlin2004",
      "repository": "yuanlin2004/chisel-spark-recipes",
      "revision": "84feafddff1e834bd16b849c827eed11676e5934",
      "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/METHOD.md",
      "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/METHOD.md",
      "document_sha256": "072fb07c8e46bb6e1e7a4c8a9416b94a87863cfc60dfde60f127e9243363dd8e",
      "checked_on": "2026-10-18"
    },
    {
      "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-server.image",
      "parent_record_id": "qwen38-chisel-dflash-replay",
      "publisher": "yuanlin2004",
      "repository": "yuanlin2004/chisel-spark-recipes",
      "revision": "84feafddff1e834bd16b849c827eed11676e5934",
      "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/server.image",
      "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/server.image",
      "document_sha256": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570",
      "checked_on": "2026-10-18"
    },
    {
      "document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
      "parent_record_id": "qwen38-chisel-dflash-replay",
      "publisher": "yuanlin2004",
      "repository": "yuanlin2004/chisel-spark-recipes",
      "revision": "84feafddff1e834bd16b849c827eed11676e5934",
      "document_path": "results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json",
      "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json",
      "document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
      "checked_on": "2026-10-18"
    },
    {
      "document_id": "chisel-tools-compare-quality",
      "parent_record_id": "qwen38-chisel-dflash-replay",
      "publisher": "yuanlin2004",
      "repository": "yuanlin2004/chisel-spark-recipes",
      "revision": "84feafddff1e834bd16b849c827eed11676e5934",
      "document_path": "tools/compare_quality.py",
      "pinned_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/tools/compare_quality.py",
      "document_sha256": "ee52180fce64fa6948d7516ac42f7f5fd0ef143584d2c284d6aa19f6fdecb783",
      "checked_on": "2026-10-18"
    },
    {
      "document_id": "official-Qwen--Qwen3.8-27B",
      "parent_record_id": "qwen38-chisel-dflash-replay",
      "publisher": "Qwen",
      "repository": "Qwen/Qwen3.8-27B",
      "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "document_path": "README.md",
      "pinned_url": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
      "document_sha256": "57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641",
      "checked_on": "2026-10-18"
    }
  ],
  "license": {
    "declared_spdx": "Apache-2.0",
    "scope": "Repository metadata only; model conversion, base weights, plugin and engine terms retain their own scope."
  },
  "editorial": {
    "version": 1,
    "published_on": "2026-10-18",
    "modified_on": "2026-10-18",
    "byline": "Run AI Here editorial",
    "creation_method": "Local Spark Qwen draft, source-audited and edited by Run AI Here with Codex; no independent model execution.",
    "answer": "The recipe is a concrete deployment candidate, but speed and quality use different request shapes and thinking modes. The source custom quality bar passes while its non-inferiority flags are false; preserve both and leave the executed local image digest unknown.",
    "sections": [
      {
        "heading": "What is pinned and what remains unknown?",
        "paragraphs": [
          "The recipe names unsloth Qwen3.8-27B NVFP4, ten DFlash2 draft tokens, bf16 GDN state and chisel-passes 0.1.0. The measured target revision is only a short prefix. A recipe base-image digest is available, but the main tests used a local build and the saved server.image is empty; the executed image digest remains unknown."
        ],
        "citations": [
          "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/ENVIRONMENT.md"
        ]
      },
      {
        "heading": "Which speed cohort is being quoted?",
        "paragraphs": [
          "The mixed-prompt test reports median decode latency and TTFT with thinking on and medium effort, one request at a time, up to 256 output tokens and two fresh-start sessions. Local llama-benchy results instead use 16K retained context, 128 output tokens and prefix cache on; its two-user throughput is aggregate, not per user."
        ],
        "citations": [
          "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/METHOD.md"
        ]
      },
      {
        "heading": "How should the quality gate be read?",
        "paragraphs": [
          "Quality tests use thinking off and compare four tasks against a separately recorded BF16 reference. The custom bar passes, but all four non_inferior flags and the overall all_non_inferior flag are false. Preserve both gates, the raw scores, sample counts, paired differences and intervals. These results do not establish equivalence or general quality preservation."
        ],
        "citations": [
          "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json"
        ]
      },
      {
        "heading": "How were the intervals computed?",
        "paragraphs": [
          "The comparison code uses matched per-question differences and a normal approximation: mean difference plus or minus 1.96 sample standard errors. It is not a bootstrap interval. The non-inferiority flag requires the lower bound above a loss of one point, while the author custom bar uses score recovery, a clearly-worse check and answer-length limits."
        ],
        "citations": [
          "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/tools/compare_quality.py"
        ]
      },
      {
        "heading": "What should an operator verify next?",
        "paragraphs": [
          "The configured 262144 context cap and four-sequence capacity are launch settings. They do not establish simultaneous near-limit fit. Verify the exact quantized artifact, drafter, plugin and image, then use representative requests and correctness tasks. Do not transfer candidate-specific image, clock or plugin settings to the BF16 reference when they were not reported."
        ],
        "citations": [
          "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.yaml"
        ]
      }
    ],
    "facts": [],
    "reported_results": [
      {
        "label": "Chisel replay mixed-prompt median decode latency",
        "metric": "decode_latency",
        "value": 15.35,
        "unit": "ms/token",
        "statistic": "source-reported median over two fresh-start sessions",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
          "thinking": true,
          "reasoning_effort": "medium",
          "greedy": true,
          "temperature": 0,
          "top_p": null,
          "concurrency": 1,
          "input_tokens": null,
          "max_output_tokens": 256,
          "output_tokens": null,
          "prompt_count": 25,
          "prompt_mix": {
            "chat": 9,
            "code": 9,
            "JSON_schema": 6,
            "long_story": 1
          },
          "repeats_per_session": 3,
          "fresh_start_sessions": 2,
          "measurement_denominator": "(last streamed token time - first) / (output tokens - 1); median latency, not whole-request served rate"
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L9",
        "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321",
        "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md",
        "source_line": 9
      },
      {
        "label": "Chisel replay mixed-prompt median TTFT",
        "metric": "ttft",
        "value": 241,
        "unit": "ms",
        "statistic": "source-reported median on the same prompt cohort",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
          "thinking": true,
          "reasoning_effort": "medium",
          "greedy": true,
          "temperature": 0,
          "top_p": null,
          "concurrency": 1,
          "input_tokens": null,
          "max_output_tokens": 256,
          "output_tokens": null,
          "prompt_count": 25,
          "prompt_mix": {
            "chat": 9,
            "code": 9,
            "JSON_schema": 6,
            "long_story": 1
          },
          "repeats_per_session": 3,
          "fresh_start_sessions": 2,
          "measurement_denominator": "request start to first streamed token"
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L10",
        "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321",
        "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md",
        "source_line": 10
      },
      {
        "label": "Local llama-benchy 16K C1",
        "metric": "generation_throughput",
        "value": 30.0,
        "unit": "tokens/s",
        "statistic": "source-reported summary; separate from mixed-prompt median latency",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
          "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores",
          "concurrency": 1,
          "input_tokens": 2048,
          "retained_context_tokens": 16384,
          "output_tokens": 128,
          "prefix_cache": true,
          "repeats_per_session": 5,
          "fresh_start_sessions": 2,
          "thinking": null,
          "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2"
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11",
        "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321",
        "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md",
        "source_line": 11
      },
      {
        "label": "Local llama-benchy 16K C2",
        "metric": "generation_throughput",
        "value": 58.0,
        "unit": "tokens/s",
        "statistic": "source-reported summary; separate from mixed-prompt median latency",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "2026-10-15 source cohort; some recorded sessions cross UTC midnight",
          "benchmark": "llama-benchy 0.4.0, local execution of spark-arena-v2 shape; not arena scores",
          "concurrency": 2,
          "input_tokens": 2048,
          "retained_context_tokens": 16384,
          "output_tokens": 128,
          "prefix_cache": true,
          "repeats_per_session": 5,
          "fresh_start_sessions": 2,
          "thinking": null,
          "measurement_denominator": "source local llama-benchy generation throughput; total across users for C2"
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/recipes/qwen3.8-27b/chisel-nvfp4-dflash2-replay.md#L11",
        "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321",
        "source_document_id": "chisel-recipes-qwen3.8-27b-chisel-nvfp4-dflash2-replay.md",
        "source_line": 11
      },
      {
        "label": "gsm8k BF16 reference",
        "metric": "task_score",
        "value": 90.06823351023503,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "Qwen/Qwen3.8-27B",
          "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reported_model_revision_prefix": null,
          "system": null,
          "node_count": null,
          "driver": null,
          "GPU_clock_cap_MHz": null,
          "engine": null,
          "executed_engine_version": null,
          "runtime_execution_revision": null,
          "recipe_base_image_digest": null,
          "executed_local_image_digest": null,
          "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.",
          "plugin": null,
          "draft_model": null,
          "draft_weight_sha256": null,
          "draft_tokens": null,
          "GDN_state_dtype": null,
          "KV_dtype": null,
          "configured_context_limit_tokens": null,
          "configured_max_requests": null,
          "GPU_memory_utilization": null,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "BF16 reference",
          "task": "gsm8k",
          "n": 1319,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "5-shot accuracy",
          "paired_difference_normal_CI95_points": [
            -1.8137451343135735,
            0.6007049523575464
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L8",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 8
      },
      {
        "label": "gsm8k NVFP4 replay",
        "metric": "task_score",
        "value": 89.46171341925702,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "NVFP4 replay",
          "task": "gsm8k",
          "n": 1319,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "5-shot accuracy",
          "paired_difference_normal_CI95_points": [
            -1.8137451343135735,
            0.6007049523575464
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L9",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 9
      },
      {
        "label": "ifeval BF16 reference",
        "metric": "task_score",
        "value": 80.96118299445472,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "Qwen/Qwen3.8-27B",
          "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reported_model_revision_prefix": null,
          "system": null,
          "node_count": null,
          "driver": null,
          "GPU_clock_cap_MHz": null,
          "engine": null,
          "executed_engine_version": null,
          "runtime_execution_revision": null,
          "recipe_base_image_digest": null,
          "executed_local_image_digest": null,
          "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.",
          "plugin": null,
          "draft_model": null,
          "draft_weight_sha256": null,
          "draft_tokens": null,
          "GDN_state_dtype": null,
          "KV_dtype": null,
          "configured_context_limit_tokens": null,
          "configured_max_requests": null,
          "GPU_memory_utilization": null,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "BF16 reference",
          "task": "ifeval",
          "n": 541,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "prompt-level strict",
          "paired_difference_normal_CI95_points": [
            -1.560611007417151,
            3.409039842906985
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L26",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 26
      },
      {
        "label": "ifeval NVFP4 replay",
        "metric": "task_score",
        "value": 81.88539741219964,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "NVFP4 replay",
          "task": "ifeval",
          "n": 541,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "prompt-level strict",
          "paired_difference_normal_CI95_points": [
            -1.560611007417151,
            3.409039842906985
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L27",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 27
      },
      {
        "label": "humaneval_chat BF16 reference",
        "metric": "task_score",
        "value": 95.1219512195122,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "Qwen/Qwen3.8-27B",
          "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reported_model_revision_prefix": null,
          "system": null,
          "node_count": null,
          "driver": null,
          "GPU_clock_cap_MHz": null,
          "engine": null,
          "executed_engine_version": null,
          "runtime_execution_revision": null,
          "recipe_base_image_digest": null,
          "executed_local_image_digest": null,
          "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.",
          "plugin": null,
          "draft_model": null,
          "draft_weight_sha256": null,
          "draft_tokens": null,
          "GDN_state_dtype": null,
          "KV_dtype": null,
          "configured_context_limit_tokens": null,
          "configured_max_requests": null,
          "GPU_memory_utilization": null,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "BF16 reference",
          "task": "humaneval_chat",
          "n": 164,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "chat pass@1",
          "paired_difference_normal_CI95_points": [
            -2.6839968720184646,
            1.4644846768965132
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L44",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 44
      },
      {
        "label": "humaneval_chat NVFP4 replay",
        "metric": "task_score",
        "value": 94.51219512195121,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "NVFP4 replay",
          "task": "humaneval_chat",
          "n": 164,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "chat pass@1",
          "paired_difference_normal_CI95_points": [
            -2.6839968720184646,
            1.4644846768965132
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L45",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 45
      },
      {
        "label": "longbench_narrativeqa BF16 reference",
        "metric": "task_score",
        "value": 34.83234254995091,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "Qwen/Qwen3.8-27B",
          "model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reported_model_revision_prefix": null,
          "system": null,
          "node_count": null,
          "driver": null,
          "GPU_clock_cap_MHz": null,
          "engine": null,
          "executed_engine_version": null,
          "runtime_execution_revision": null,
          "recipe_base_image_digest": null,
          "executed_local_image_digest": null,
          "executed_image_boundary": "Reference model and bf16 KV are reported; candidate-specific image, plugin and clock settings are not imputed to this arm.",
          "plugin": null,
          "draft_model": null,
          "draft_weight_sha256": null,
          "draft_tokens": null,
          "GDN_state_dtype": null,
          "KV_dtype": null,
          "configured_context_limit_tokens": null,
          "configured_max_requests": null,
          "GPU_memory_utilization": null,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "BF16 reference",
          "task": "longbench_narrativeqa",
          "n": 200,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "F1, full stories",
          "paired_difference_normal_CI95_points": [
            -1.6421295670363527,
            2.4483306144283086
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L62",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 62
      },
      {
        "label": "longbench_narrativeqa NVFP4 replay",
        "metric": "task_score",
        "value": 35.235443073646884,
        "unit": "score points",
        "statistic": "source paired-comparison raw score; retain task-specific definition",
        "uncertainty": null,
        "conditions": {
          "model": "Qwen3.8-27B",
          "model_artifact": "unsloth/Qwen3.8-27B-NVFP4",
          "model_revision": null,
          "reported_model_revision_prefix": "f0b7c9e7",
          "system": "1 x DGX Spark GB10 sm_121; 128 GB unified memory",
          "node_count": 1,
          "driver": "580.178.04",
          "GPU_clock_cap_MHz": 2000,
          "engine": "vLLM",
          "executed_engine_version": "0.31.1rc1.dev380",
          "runtime_execution_revision": null,
          "recipe_base_image_digest": "sha256:b77010699ec42ab88b3409f4dd101f2288dfd8895e2778e5bb737e03a20953f0",
          "executed_local_image_digest": null,
          "executed_image_boundary": "benchmark and quality used a local build with the same plugin code preinstalled; saved server.image is []; do not substitute the recipe base digest as the executed image digest",
          "plugin": "chisel-passes 0.1.0",
          "draft_model": "yuanlin2004/Qwen3.8-27B-Chisel-DFlash2-NVFP4-drafter",
          "draft_weight_sha256": "8d464c9f597c41a2e8917c585454239be14f254592f114675fd9d5d5dbc92f4c",
          "draft_tokens": 10,
          "GDN_state_dtype": "bfloat16",
          "KV_dtype": null,
          "configured_context_limit_tokens": 262144,
          "configured_max_requests": 4,
          "GPU_memory_utilization": 0.6,
          "experiment_date": "reference cohort 2026-10-09; candidate cohort filed 2026-10-15; compare file retains earlier original paths",
          "quality_arm": "NVFP4 replay",
          "task": "longbench_narrativeqa",
          "n": 200,
          "thinking": false,
          "greedy": true,
          "concurrency": null,
          "input_tokens": null,
          "output_tokens": null,
          "benchmark": "lm-evaluation-harness 0.4.13; chat template; full task",
          "reference_model": "Qwen/Qwen3.8-27B",
          "reference_model_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
          "reference_weight_precision": "bfloat16",
          "reference_KV_dtype": "bfloat16",
          "metric_definition": "F1, full stories",
          "paired_difference_normal_CI95_points": [
            -1.6421295670363527,
            2.4483306144283086
          ],
          "non_inferior": false,
          "author_custom_bar_passed": true
        },
        "evidence_state": "source_reported",
        "site_reproduced": false,
        "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L63",
        "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
        "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
        "source_line": 63
      }
    ],
    "source_document_sha256": "5da566bc197bf4ba29c7c4d343b3c0362577de023425369a1a6ea9e53de95321"
  },
  "quality_comparisons": [
    {
      "task": "gsm8k",
      "n": 1319,
      "reference_score": 90.06823351023503,
      "candidate_score": 89.46171341925702,
      "diff_points": -0.6065200909780136,
      "ci95_points": [
        -1.8137451343135735,
        0.6007049523575464
      ],
      "non_inferior": false,
      "length_ratio": 1.0086485431409682,
      "flips_right_to_wrong": 37,
      "flips_wrong_to_right": 29,
      "recovery": 0.9932659932659933,
      "clearly_worse": false,
      "length_ok": true,
      "passes": true,
      "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap",
      "evidence_state": "source_reported",
      "site_reproduced": false,
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L6",
      "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
      "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
      "source_line": 6
    },
    {
      "task": "ifeval",
      "n": 541,
      "reference_score": 80.96118299445472,
      "candidate_score": 81.88539741219964,
      "diff_points": 0.9242144177449169,
      "ci95_points": [
        -1.560611007417151,
        3.409039842906985
      ],
      "non_inferior": false,
      "length_ratio": 1.001384198457369,
      "flips_right_to_wrong": 21,
      "flips_wrong_to_right": 26,
      "recovery": 1.0114155251141552,
      "clearly_worse": false,
      "length_ok": true,
      "passes": true,
      "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap",
      "evidence_state": "source_reported",
      "site_reproduced": false,
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L24",
      "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
      "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
      "source_line": 24
    },
    {
      "task": "humaneval_chat",
      "n": 164,
      "reference_score": 95.1219512195122,
      "candidate_score": 94.51219512195121,
      "diff_points": -0.6097560975609756,
      "ci95_points": [
        -2.6839968720184646,
        1.4644846768965132
      ],
      "non_inferior": false,
      "length_ratio": 1.0066489055410106,
      "flips_right_to_wrong": 2,
      "flips_wrong_to_right": 1,
      "recovery": 0.9935897435897434,
      "clearly_worse": false,
      "length_ok": true,
      "passes": true,
      "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap",
      "evidence_state": "source_reported",
      "site_reproduced": false,
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L42",
      "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
      "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
      "source_line": 42
    },
    {
      "task": "longbench_narrativeqa",
      "n": 200,
      "reference_score": 34.83234254995091,
      "candidate_score": 35.235443073646884,
      "diff_points": 0.403100523695978,
      "ci95_points": [
        -1.6421295670363527,
        2.4483306144283086
      ],
      "non_inferior": false,
      "length_ratio": 0.9618899475014583,
      "recovery": 1.011572593003698,
      "clearly_worse": false,
      "length_ok": true,
      "passes": true,
      "interval_method": "paired per-question differences; mean +/- 1.96 * sample SD / sqrt(n); normal approximation, not bootstrap",
      "evidence_state": "source_reported",
      "site_reproduced": false,
      "evidence_url": "https://github.com/yuanlin2004/chisel-spark-recipes/blob/84feafddff1e834bd16b849c827eed11676e5934/results/qwen3.8-27b/chisel-nvfp4-dflash2-replay/2026-10-15/quality/compare-vs-bf16.json#L60",
      "source_document_sha256": "46aec52dd7d9d842b58c710904f6af19960f1196a7f05f2c834f4532481121fb",
      "source_document_id": "chisel-results-qwen3.8-27b-chisel-nvfp4-dflash2-replay-2026-10-15-quality-compare-vs-bf16.json",
      "source_line": 60
    }
  ]
}