# Kimi IQ1_S on Three Sparks: Scores, Retries and Missing Pins Audit a publisher-labelled K3 layer-split deployment, approximate memory and small GSM8K/HumanEval subsets, retaining first-pass failures and timeout retries. - Type: Performance report - Evidence: source_reported; site reproduced: false - Model family: Kimi - Model: k3-neuron-iq1s, nine GGUF shards - Original author: chishiki37 - Original project: https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark - Pinned document: https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md - Source revision: de1fa14ea798512f54dafc8b8795eb3ef1a6c367 - Source commit time (not experiment date): 2026-08-07T15:45:04Z - Checked: 2026-10-18 - Experiment date: 2026-08-07 - Runtime topics: llama.cpp - Hardware topics: spark, nvidia - Reported setup: {"system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "kv_dtype": "f16", "context_limit_tokens": 1048576, "experiment_date": "2026-08-07"} - Performance figures: attributed selections with per-point conditions; no cross-source ranking ## Conditions Unknown is null. Recipe options are not measured benchmark settings. ```json { "conditions": { "model_revision": null, "weight_hash": null, "runtime_execution_revision": null, "input_tokens": null, "output_tokens": null, "concurrency": null, "cache_state": null, "quantization": null, "speculative_decoding": null, "temperature": null, "top_p": null }, "test_profiles": [ { "label": "steady decode", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L43", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "steady decode", "input_length_source_label": null } }, { "label": "8K prefill", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L44", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "8K prefill", "input_length_source_label": "8K" } }, { "label": "32K prefill", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L45", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "32K prefill", "input_length_source_label": "32K" } }, { "label": "131K prefill", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L46", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "131K prefill", "input_length_source_label": "131K" } }, { "label": "GSM8K local subset", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-local.json#L5", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "GSM8K canonical test subset", "correct": 49, "total": 50, "subset_ids": "50 source-indexed questions; fixed subset, not full benchmark", "timeout_seconds": 1800, "retry_policy": "GLM comparison changed output cap; local retries unreported" } }, { "label": "HumanEval initial local subset", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-p18081.json#L6", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval subset IDs 0 through 19", "correct": 14, "total": 20, "subset_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], "timeout_seconds": 600, "timeouts": 4, "retry_policy": "initial run; four timeouts retained as failures" } }, { "label": "HumanEval timeout retries", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-retry-p18081.json#L6", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval timeout-only subset", "correct": 4, "total": 4, "subset_ids": [ 1, 6, 10, 17 ], "timeout_seconds": null, "retry_policy": "rerun only four timed-out problems with higher limits; not fresh 20-problem pass" } }, { "label": "HumanEval local after selected retries", "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L83", "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval subset IDs 0 through 19", "correct": 18, "total": 20, "subset_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], "timeout_seconds": null, "retry_policy": "14 initial passes plus 4 successful timeout-only retries; not first-pass 18/20", "failed_ids": [ 2, 7 ] } } ], "recipe_parameters": null, "deployment_options": [] } ``` ## Reading notes - A source document and its selected result points are not independent experiment counts. ## Limits - No independent Run AI Here model test. - Executed identity, timing definition, cache state, repetition unit and missingness must be read per result. - No cross-hardware or unmatched model score ranking. ## License declaration {"declared_spdx": null, "scope": "Attribution and finite factual interpretation only; models and dependencies retain their own terms."} ## Editorial answer The source reports 49/50 on a GSM8K subset and a HumanEval composite of 18/20 after retrying four timed-out problems. The initial HumanEval result was 14/20. Unequal cloud sampling, unpinned weight identity and small subsets do not prove lossless IQ1_S quantization or full 1M-input capability. By Run AI Here editorial · 2026-10-18 Local Spark Qwen writing draft; Codex source audit and rewrite. No site model benchmark. ## Which identity and topology are known? The August 7 publisher labels the artifact Kimi K3/Kimi-Linear IQ1_S with nine GGUF shards. No exact upstream checkpoint revision or weight hash is given, so the label is not an independently verified official artifact. Three DGX Sparks use layer-split RPC through a pwilkin llama.cpp fork, not TP3; the actual transport configuration uses MTU4096, while the fabric overview separately says MTU9000. Sources: https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md ## What is measured versus configured? The report gives approximate steady decode and prefill summaries through a 131K-labelled input. The 1048576-token window and one slot are configured limits. About 119–120 GB per node is an approximate occupancy report with no peak-telemetry method; it cannot establish peak headroom at a completed million-token input. Exact engine commit and runtime image are also absent. Sources: https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md ## How are scores and retries retained? The local GSM8K subset has 49 correct of 50. Raw HumanEval results show 14 correct of 20 initially, including four 600-second timeouts. The selected timeout-only rerun passes all four, producing the reported composite 18/20. The two remaining assertion failures are IDs2 and7. Do not turn the composite into a first-pass result or call every failure a floating-point issue. Sources: https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-local.json, https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-p18081.json, https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-retry-p18081.json ## Why is this not a quantization equivalence test? Cloud Kimi uses temperature1 and a smaller GSM8K output cap while local uses temperature0.2; the GLM comparison raises its cap after output exhaustion. These are small fixed subsets with no unquantized local control. Matching selected scores and failure IDs does not prove equivalence, zero quantization cost or a general model ranking. No standardized full-suite leaderboard score is created here. Sources: https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md, https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/cloud-humaneval-logic-k3-cloud.json, https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-cloud.json ## Selected source-reported figures Each point retains its own configuration, measurement unit and source. No site reproduction or cross-source ranking. ```json [ { "label": "steady decode", "metric": "decode_rate", "value": 2.9, "unit": "tokens/s", "statistic": "approximate source summary", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "steady decode", "input_length_source_label": null }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 43, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L43" }, { "label": "8K prefill", "metric": "prefill_rate", "value": 79.6, "unit": "tokens/s", "statistic": "source summary; exact per-run token counts not supplied", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "8K prefill", "input_length_source_label": "8K" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 44, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L44" }, { "label": "32K prefill", "metric": "prefill_rate", "value": 43.9, "unit": "tokens/s", "statistic": "source summary; exact per-run token counts not supplied", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "32K prefill", "input_length_source_label": "32K" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 45, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L45" }, { "label": "131K prefill", "metric": "prefill_rate", "value": 48.5, "unit": "tokens/s", "statistic": "source summary; exact per-run token counts not supplied", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": null, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": null, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "workload": "131K prefill", "input_length_source_label": "131K" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 46, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L46" }, { "label": "GSM8K local subset", "metric": "accuracy", "value": 98.0, "unit": "percent", "statistic": "source fixed-subset score, no confidence interval", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "GSM8K canonical test subset", "correct": 49, "total": 50, "subset_ids": "50 source-indexed questions; fixed subset, not full benchmark", "timeout_seconds": 1800, "retry_policy": "GLM comparison changed output cap; local retries unreported" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-extra0", "source_document_sha256": "2d297e3f59c7beb393fa72e6297edec8af0b6525139057ee0e26b425d071f2be", "source_line": 5, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-local.json#L5" }, { "label": "HumanEval initial local subset", "metric": "accuracy", "value": 70.0, "unit": "percent", "statistic": "source fixed-subset score, no confidence interval", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval subset IDs 0 through 19", "correct": 14, "total": 20, "subset_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], "timeout_seconds": 600, "timeouts": 4, "retry_policy": "initial run; four timeouts retained as failures" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-extra1", "source_document_sha256": "29a942e4a2194c737b8492b59ab752015896fc8e3180cc85135c33c6b249f590", "source_line": 6, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-p18081.json#L6" }, { "label": "HumanEval timeout retries", "metric": "accuracy", "value": 100.0, "unit": "percent", "statistic": "source fixed-subset score, no confidence interval", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval timeout-only subset", "correct": 4, "total": 4, "subset_ids": [ 1, 6, 10, 17 ], "timeout_seconds": null, "retry_policy": "rerun only four timed-out problems with higher limits; not fresh 20-problem pass" }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-extra2", "source_document_sha256": "469ff6a70092ca219c70e3dc2d6967729133f74e97ed0b9cd56e68323b2f2053", "source_line": 6, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-retry-p18081.json#L6" }, { "label": "HumanEval local after selected retries", "metric": "accuracy", "value": 90.0, "unit": "percent", "statistic": "source composite score after retries", "uncertainty": null, "conditions": { "model": "Kimi K3 IQ1_S (publisher label; exact upstream checkpoint not pinned)", "model_artifact": "k3-neuron-iq1s, nine GGUF shards", "model_revision": null, "runtime_execution_revision": null, "executed_image_digest": null, "engine": "pwilkin llama.cpp fork, build-rdma", "system": "three DGX Spark GB10, 128 GB unified memory each", "node_count": 3, "parallelism": "layer-split RPC; two remote workers, not TP3", "weight_precision": "IQ1_S", "kv_dtype": "f16", "context_limit_tokens": 1048576, "configured_max_requests": 1, "concurrency": 1, "load_mode": "none", "actual_transport_MTU": 4096, "fabric_overview_MTU": 9000, "temperature": 0.2, "top_p": null, "thinking": null, "input_tokens": null, "output_tokens": 8192, "actual_output_tokens": null, "cache_state": null, "repetitions": null, "experiment_date": "2026-08-07", "peak_runtime_RAM": null, "benchmark": "HumanEval subset IDs 0 through 19", "correct": 18, "total": 20, "subset_ids": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], "timeout_seconds": null, "retry_policy": "14 initial passes plus 4 successful timeout-only retries; not first-pass 18/20", "failed_ids": [ 2, 7 ] }, "evidence_state": "source_reported", "site_reproduced": false, "source_document_id": "kimi-iq1s-readme", "source_document_sha256": "4bccb5ffc313f08ca53f62b6c8fc880c1317c07f79fed53c522bfc8e99bea20d", "source_line": 83, "evidence_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/README.md#L83" } ] ``` ## Supplemental source identities ```json { "supplemental_sources": [ { "document_id": "kimi-iq1s-extra0", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/canon-gsm8k-k3-local.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-local.json", "document_sha256": "2d297e3f59c7beb393fa72e6297edec8af0b6525139057ee0e26b425d071f2be", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra1", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/humaneval-p18081.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-p18081.json", "document_sha256": "29a942e4a2194c737b8492b59ab752015896fc8e3180cc85135c33c6b249f590", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra2", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/humaneval-retry-p18081.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/humaneval-retry-p18081.json", "document_sha256": "469ff6a70092ca219c70e3dc2d6967729133f74e97ed0b9cd56e68323b2f2053", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra3", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/cloud-humaneval-logic-k3-cloud.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/cloud-humaneval-logic-k3-cloud.json", "document_sha256": "8d42fd9033e794d5a8a7330d5c56707128b521f869c46be071a364c5f45fa22c", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" }, { "document_id": "kimi-iq1s-extra4", "parent_record_id": "kimi-k3-iq1s-three-spark-scores", "publisher": "chishiki37", "repository": "chishiki37/kimi-k3-iq1s-3node-dgx-spark", "revision": "de1fa14ea798512f54dafc8b8795eb3ef1a6c367", "document_path": "benchmarks/canon-gsm8k-k3-cloud.json", "pinned_url": "https://github.com/chishiki37/kimi-k3-iq1s-3node-dgx-spark/blob/de1fa14ea798512f54dafc8b8795eb3ef1a6c367/benchmarks/canon-gsm8k-k3-cloud.json", "document_sha256": "636e0ee2c31986d961ffe637927bcb95aa1d3d82d78dea08f7efe0807f29971b", "last_commit_at": "2026-08-07T15:45:04Z", "checked_on": "2026-10-18", "public_URL_is_immutable": true, "revision_kind": "Git document revision; not necessarily the executed runtime or weights" } ] } ``` ## Paired source quality comparisons ```json { "quality_comparisons": [ { "benchmark": "GSM8K", "subset_size": 50, "local_correct": 49, "cloud_Kimi_correct": 49, "cloud_GLM52_correct": 49, "local_temperature": 0.2, "cloud_Kimi_temperature": 1.0, "local_output_cap": 8192, "cloud_Kimi_output_cap": 2048, "GLM_cap_changed_after_exhaustion": true, "matched_accuracy_proves_equivalence": false }, { "benchmark": "HumanEval", "subset_size": 20, "initial_local_correct": 14, "initial_timeouts": 4, "retry_subset_ids": [ 1, 6, 10, 17 ], "retry_correct": 4, "final_local_correct": 18, "remaining_failure_ids": [ 2, 7 ], "first_pass_18_of_20": false } ] } ``` ## memory observations ```json { "memory_observations": [ { "kind": "approximate_per_node_occupancy_unknown_measurement_method", "source_range_GB": [ 119, 120 ], "unit": "GB (source label; byte convention unspecified)", "node_count": 3, "is_measured_peak": false, "peak_RAM": null } ] } ``` ## long context cases ```json { "long_context_cases": [ { "classification": "configured_context_not_completed_full_window", "configured_limit_tokens": 1048576, "longest_performance_source_label": "131K", "actual_full_window_prompt_tokens": null } ] } ``` ## quality comparisons ```json { "quality_comparisons": [ { "benchmark": "GSM8K", "subset_size": 50, "local_correct": 49, "cloud_Kimi_correct": 49, "cloud_GLM52_correct": 49, "local_temperature": 0.2, "cloud_Kimi_temperature": 1.0, "local_output_cap": 8192, "cloud_Kimi_output_cap": 2048, "GLM_cap_changed_after_exhaustion": true, "matched_accuracy_proves_equivalence": false }, { "benchmark": "HumanEval", "subset_size": 20, "initial_local_correct": 14, "initial_timeouts": 4, "retry_subset_ids": [ 1, 6, 10, 17 ], "retry_correct": 4, "final_local_correct": 18, "remaining_failure_ids": [ 2, 7 ], "first_pass_18_of_20": false } ] } ```