# GLM Flash Scores: Historical NVFP4 and the EXL3 Retraction — historical NVFP4 Separate a full historical MMLU run and small task subsets from the corrected EXL3 router stack, whose old task scores were withdrawn and await reruns. - Type: Performance report - Evidence: source_reported; site reproduced: false - Model family: GLM - Model: local-inference-lab/GLM-5.3-Flash-NVFP4 - Original author: NNNtrance - Original project: https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark - Pinned document: https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md - Source revision: eb11cef295c107c5bd21402106eacea5ebd1fbf3 - Source commit time (not experiment date): 2026-09-09T21:23:55Z - Checked: 2026-10-18 - Experiment date: 2026-09-03 09:08-18:08 UTC - Runtime topics: vLLM - Hardware topics: spark, nvidia - Reported setup: {"system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "kv_dtype": "FP8", "context_limit_tokens": 1000000, "experiment_date": "2026-09-03 09:08-18:08 UTC"} - Performance figures: attributed selections with per-point conditions; no cross-source ranking ## Conditions Unknown is null. Recipe options are not measured benchmark settings. ```json { "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "test_profiles": [ { "label": "Tool-eval eight-trial mean", "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L47", "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "tool-eval-bench 2.6.1.dev39+gd3352edf5", "scenarios": 88, "trials": 8, "seed": 42, "max_turns": 8, "concurrency": 1, "source_trial_SD": 0.9, "source_CI95": [ 87.1, 88.2 ], "CI_method_independently_verified": false } }, { "label": "Tool-eval Pass^8", "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L56", "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "tool-eval-bench", "scenarios": 88, "trials": 8, "concurrency": 1 } }, { "label": "GSM8K 200-question subset", "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L197", "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "GSM8K", "subset_size": 200, "full_test_size": 1319, "correct": 188, "shots": 8, "CoT": true, "concurrency": null } }, { "label": "IFEval prompt strict, custom evaluator", "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L155", "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "IFEval", "prompts": 541, "correct": 427, "concurrency": null, "evaluator": "custom; not independently cross-checked", "proven_lower_bound": false } }, { "label": "IFEval instruction level, custom evaluator", "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L156", "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "IFEval", "instructions": 834, "correct": 710, "concurrency": null, "evaluator": "custom; not independently cross-checked", "proven_lower_bound": false } }, { "label": "Needle four sizes x five depths", "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L221", "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "single-fact NIAH", "correct": 20, "total": 20, "max_actual_input_tokens": 997952, "concurrency": null, "initial_timeout_seconds": 120, "rerun_timeout_seconds": 3600, "identical_content_seed_on_rerun": true, "multihop_validated": false } }, { "label": "Full MMLU exact accuracy", "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json#L4", "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": null, "thinking": null, "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "MMLU", "subset_size": 14042, "subjects": 57, "shots": 0, "scoring": "loglikelihood, no generation or sampling", "harness": "lm-eval 0.4.9", "concurrency": 8, "timeout_seconds": 36000, "max_retries": 6, "accuracy_stderr": 0.0028276391244761038, "stderr_is_SD": false, "initial_harness_timeout_at_fraction": 0.81, "engine_errors_reported": 0, "rerun_same_content": true } } ], "recipe_parameters": null, "deployment_options": [] } ``` ## Reading notes - Selected values are source excerpts, not independent experiment counts. ## Limits - No independent Run AI Here model test. - Read source date, exact execution identity, denominator, memory scope and retries per observation. - No unmatched quality ranking or hardware speed ranking. ## License declaration {"declared_spdx": null, "scope": "Attribution and finite original factual interpretation only; models and dependencies retain their own terms."} ## Editorial answer The September NVFP4 report and October EXL3 correction are different deployment lines. The EXL3 author withdrew its old task scores after finding an unloaded router bias. Current evidence is a same-quant probability check; it does not revalidate old task scores or import the NVFP4 task panel. By Run AI Here editorial · 2026-10-18 Local Spark Qwen draft; Codex source audit and rewrite. No site model benchmark. ## Which scores remain historical NVFP4 observations? The September 3 TP3+EP Marlin NVFP4 t10 campaign uses GMU 0.85, FP8 KV, DFlash2 k7 and a decimal-million configured window. Full MMLU is 14042 zero-shot loglikelihood questions with exact accuracy 0.8594217347956131 and harness SE 0.0028276391244761038. Tool-eval has eight trials over 88 hard scenarios, mean 87.8 and source SD 0.9. GSM8K is only 200 questions, 8-shot,188 correct. IFEval uses a custom unverified evaluator. Sources: https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md, https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json, https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/tool-eval-bench/tool-eval-bench-8-trials.json ## Why did the EXL3 author withdraw its old scores? The October 8 notice identifies missing e_score_correction_bias in the old cuda-exl3 plugin. Wrong expert routing invalidates that repository's old task panels. The new TP3 main stack loads the bias and fails boot if missing. TP2 remains unported and untested. This notice is scoped to the EXL3 plugin line, not proof of the same bug in the separate historical NVFP4 line. Sources: https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/README.md, https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/tracks/tp3-main/README.md, https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/quality-gates.md ## What was rerun on corrected EXL3? The detailed KL table compares the same EXL3 checkpoint to exllamav3 1.5.4 on six author-written texts. Assistant KL is 0.1033 pre-fix and 0.00828 on production a4i. Perplexity 2.222 belongs to bias-fix a4g; production perplexity is not reported in that table. The reference is quantized, not BF16. Task benchmarks have not been rerun, and the prepared text set and per-position logits are not public. Sources: https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md, https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/quality-gates.md ## How are failed attempts kept? Historical NVFP4 MMLU initially hits a harness queue timeout, and needle tests initially exceed a 120s request cap; same-content reruns use longer timeouts with zero engine errors reported. These are harness retries, not model-accuracy zeros or engine recoveries. A separate model line's completed task scores cannot fill the corrected EXL3 missing scores. Sources: https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md, https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/needle/needle-in-a-haystack-report.md, https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/quality-gates.md ## Selected source-reported figures Each point retains its own configuration, measurement unit and source. No site reproduction or cross-source ranking. ```json [ { "label": "Tool-eval eight-trial mean", "metric": "tool_eval_score", "value": 87.8, "unit": "points", "statistic": "mean across eight trials; source SD 0.9, source CI [87.1,88.2]", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "tool-eval-bench 2.6.1.dev39+gd3352edf5", "scenarios": 88, "trials": 8, "seed": 42, "max_turns": 8, "concurrency": 1, "source_trial_SD": 0.9, "source_CI95": [ 87.1, 88.2 ], "CI_method_independently_verified": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 47, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L47" }, { "label": "Tool-eval Pass^8", "metric": "all_trials_pass_rate", "value": 76.1, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "tool-eval-bench", "scenarios": 88, "trials": 8, "concurrency": 1 }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 56, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L56" }, { "label": "GSM8K 200-question subset", "metric": "accuracy", "value": 94.0, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "GSM8K", "subset_size": 200, "full_test_size": 1319, "correct": 188, "shots": 8, "CoT": true, "concurrency": null }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 197, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L197" }, { "label": "IFEval prompt strict, custom evaluator", "metric": "accuracy", "value": 78.9, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "IFEval", "prompts": 541, "correct": 427, "concurrency": null, "evaluator": "custom; not independently cross-checked", "proven_lower_bound": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 155, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L155" }, { "label": "IFEval instruction level, custom evaluator", "metric": "accuracy", "value": 85.1, "unit": "percent", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "IFEval", "instructions": 834, "correct": 710, "concurrency": null, "evaluator": "custom; not independently cross-checked", "proven_lower_bound": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 156, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L156" }, { "label": "Needle four sizes x five depths", "metric": "retrieval_correct_count", "value": 20, "unit": "correct probes", "statistic": "single finite benchmark; no dispersion reported", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": 0, "thinking": "on, effort low except MMLU", "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "single-fact NIAH", "correct": 20, "total": 20, "max_actual_input_tokens": 997952, "concurrency": null, "initial_timeout_seconds": 120, "rerun_timeout_seconds": 3600, "identical_content_seed_on_rerun": true, "multihop_validated": false }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 221, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L221" }, { "label": "Full MMLU exact accuracy", "metric": "accuracy", "value": 0.8594217347956131, "unit": "fraction", "statistic": "single full MMLU, harness standard error 0.0028276391244761038", "uncertainty": null, "conditions": { "model": "GLM-5.3-Flash", "model_artifact": "local-inference-lab/GLM-5.3-Flash-NVFP4", "model_revision": null, "engine": "LIL vLLM 0.1.dev0+lil.jovian.9c4dd0548, harem/glm53-lil:t10", "runtime_execution_revision": null, "executed_image_digest": null, "system": "three DGX Spark GB10, TP3 + EP", "node_count": 3, "parallelism": "TP3+EP", "weight_precision": "NVFP4 storage / Marlin MoE", "kv_dtype": "FP8", "draft": "DFlash2 k7", "context_limit_tokens": 1000000, "memory_utilization": 0.85, "configured_max_requests": 8, "batch_tokens": 2048, "experiment_date": "2026-09-03 09:08-18:08 UTC", "status": "historical NVFP4 line, maintenance stopped September 6; not corrected EXL3 scores", "host_peak_RAM": null, "temperature": null, "thinking": null, "input_tokens": null, "output_tokens": null, "cache_state": null, "benchmark": "MMLU", "subset_size": 14042, "subjects": 57, "shots": 0, "scoring": "loglikelihood, no generation or sampling", "harness": "lm-eval 0.4.9", "concurrency": 8, "timeout_seconds": 36000, "max_retries": 6, "accuracy_stderr": 0.0028276391244761038, "stderr_is_SD": false, "initial_harness_timeout_at_fraction": 0.81, "engine_errors_reported": 0, "rerun_same_content": true }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "glm-tp3-mmlu", "source_document_sha256": "86ed1c5586cfe259e7ff438fff2baf1df2dd2ebc069b3db5bbce0763c9ef03c8", "source_line": 4, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json#L4" } ] ``` ## Supplemental source identities ```json { "supplemental_sources": [ { "document_id": "glm-tp3-readme", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/README.md", "document_sha256": "5912e935333340dbd77ecd01cec8c7e207990bc6f446341f741bb3bd66d6cf4b", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-memory", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "docs/05-memory-ladder.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/05-memory-ladder.md", "document_sha256": "29874450acd65f1e6614bd3fc04fe0815ef8b9036007867b8446e63ebb4092de", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-mmlu", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/mmlu/mmlu-full-lm-eval-results.json", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/mmlu/mmlu-full-lm-eval-results.json", "document_sha256": "86ed1c5586cfe259e7ff438fff2baf1df2dd2ebc069b3db5bbce0763c9ef03c8", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-tool", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/tool-eval-bench/tool-eval-bench-8-trials.json", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/tool-eval-bench/tool-eval-bench-8-trials.json", "document_sha256": "7a46e4d7b52956a0fb0662838081b526e7ceeb30e14e553648227bade4dc6df5", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-needle", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/needle/needle-in-a-haystack-report.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/needle/needle-in-a-haystack-report.md", "document_sha256": "cceef7123bbb9e9d34c2c6a44bc00055b81f896cbf51977053766e770c30ae3d", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-tp3-results", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark", "revision": "eb11cef295c107c5bd21402106eacea5ebd1fbf3", "document_path": "results/README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/results/README.md", "document_sha256": "15388126cd782399e618a52b5cbcd2c5af46f7de347f6903e4705d719f84b037", "last_commit_at": "2026-09-09T21:23:55Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-readme", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "README.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/README.md", "document_sha256": "a399f1dd57e66933e4b947eb4f14212815dc0526deee69942624759b178038d0", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-kl", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "results/main-stack/kl-health.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/kl-health.md", "document_sha256": "9fcb41cb68098fe0e982f727f6c57b86656748fd41bbd6a0f64dfc794dbb2ba1", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-gates", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "results/main-stack/quality-gates.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/results/main-stack/quality-gates.md", "document_sha256": "9d0387be075bd6c214ab8e9b2bb70a46577b5052884c06c0db02d1869b295f28", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" }, { "document_id": "glm-exl3-stack", "parent_record_id": "glm53-flash-nvfp4-tp3-scores", "publisher": "NNNtrance", "repository": "NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark", "revision": "83ed1eccd93a15ae9efe495dbee016cd57823c5f", "document_path": "docs/20-main-stack.md", "pinned_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/docs/20-main-stack.md", "document_sha256": "b7ef6f4c82dd1bfa693452cb1fcfb57c8f3270455ca894b667e24048cc20786d", "last_commit_at": "2026-10-16T01:23:58Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git source document revision; executed engine and weight revisions are separate fields" } ] } ``` ## Paired source quality comparisons ```json { "quality_comparisons": [ { "id": "NVFP4-historical-task-panel", "record_id": "glm53-flash-nvfp4-tp3-scores", "task_scores_current_EXL3": false, "task_scores_withdrawn_by_EXL3_notice": false, "source_scope": "separate NVFP4 repository, September 3 measurements", "tool_trial_scores": [ 88, 88, 86, 87, 88, 88, 88, 89 ], "tool_mean": 87.8, "source_SD": 0.9, "source_CI95": [ 87.1, 88.2 ], "MMLU_accuracy": 0.8594217347956131, "MMLU_standard_error": 0.0028276391244761038, "IFEval_evaluator_crosschecked": false, "proven_IFEval_lower_bound": false }, { "id": "EXL3-retraction-and-rerun-gap", "record_id": "glm53-flash-exl3-router-correction", "withdrawn_on": "2026-10-08", "old_task_scores_withdrawn": true, "current_task_accuracy": null, "current_task_benchmarks_rerun": false, "router_bias_loaded_after_fix": true, "production_KL": 0.00828, "production_PPL": null, "bias_fix_PPL": 2.222, "reference_PPL": 2.224, "BF16_reference": false, "public_text_set_available": false, "full_per_position_logits_public": false, "TP2_fixed_or_validated": false } ] } ``` ## memory observations ```json { "memory_observations": [ { "kind": "historical operational free host memory, not peak", "value_range": [ 6.4, 10 ], "unit": "GB, source approximate", "historical_NVFP4_only": true, "peak_RAM": null, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 28, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L28" } ] } ``` ## case reports ```json { "case_reports": [ { "id": "NVFP4-harness-only-reruns", "classification": "source_reported_harness_timeouts", "MMLU_initial_progress_fraction": 0.81, "MMLU_initial_error": "aiohttp total timeout includes queue", "MMLU_rerun_timeout_seconds": 36000, "needle_initial_timeout_seconds": 120, "needle_rerun_timeout_seconds": 3600, "same_content_and_seed": true, "reported_engine_errors": 0, "initial_partial_not_full_score": true, "source_document_id": "glm-tp3-scores", "source_document_sha256": "e60fb038ca4ab3785a68abe0cdf0906452052e091443c201f90d93ea7590a495", "source_line": 35, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-NVFP4-TP3-3x-DGX-Spark/blob/eb11cef295c107c5bd21402106eacea5ebd1fbf3/docs/06-benchmarks.md#L35" }, { "id": "EXL3-router-bias-missing", "classification": "source_reported_correctness_bug_and_retraction", "plugin_pre_fix_short": "754421f or earlier", "missing_parameter": "e_score_correction_bias", "source_scope": "EXL3 plugin line only; no evidence that separate NVFP4 line shares defect", "old_task_scores_withdrawn": true, "current_task_score": null, "source_document_id": "glm-exl3-readme", "source_document_sha256": "a399f1dd57e66933e4b947eb4f14212815dc0526deee69942624759b178038d0", "source_line": 18, "evidence_url": "https://github.com/NNNtrance/GLM-5.3-Flash-EXL3-DGX-Spark/blob/83ed1eccd93a15ae9efe495dbee016cd57823c5f/README.md#L18" } ] } ``` ## quality comparisons ```json { "quality_comparisons": [ { "id": "NVFP4-historical-task-panel", "record_id": "glm53-flash-nvfp4-tp3-scores", "task_scores_current_EXL3": false, "task_scores_withdrawn_by_EXL3_notice": false, "source_scope": "separate NVFP4 repository, September 3 measurements", "tool_trial_scores": [ 88, 88, 86, 87, 88, 88, 88, 89 ], "tool_mean": 87.8, "source_SD": 0.9, "source_CI95": [ 87.1, 88.2 ], "MMLU_accuracy": 0.8594217347956131, "MMLU_standard_error": 0.0028276391244761038, "IFEval_evaluator_crosschecked": false, "proven_IFEval_lower_bound": false }, { "id": "EXL3-retraction-and-rerun-gap", "record_id": "glm53-flash-exl3-router-correction", "withdrawn_on": "2026-10-08", "old_task_scores_withdrawn": true, "current_task_accuracy": null, "current_task_benchmarks_rerun": false, "router_bias_loaded_after_fix": true, "production_KL": 0.00828, "production_PPL": null, "bias_fix_PPL": 2.222, "reference_PPL": 2.224, "BF16_reference": false, "public_text_set_available": false, "full_per_position_logits_public": false, "TP2_fixed_or_validated": false } ] } ```