diff --git a/stacks/gen-seat/compose.yaml b/stacks/gen-seat/compose.yaml index e276322..4d05f9e 100644 --- a/stacks/gen-seat/compose.yaml +++ b/stacks/gen-seat/compose.yaml @@ -56,14 +56,11 @@ services: - float32 - --kv-cache-dtype - ${GEN_KV_CACHE_DTYPE:-fp8} - # ⚠️ PREFIX CACHING DISABLED 2026-08-17 as the MTP mitigation. vLLM V1 - # defaults prefix caching ON, so the explicit --no- form is required to - # turn it off. With MTP re-enabled below, APC-off forces the GDN cache - # into a mode where the partial-accept align-path bug is inert - # (vllm#43559 / #47194): the corruption is gated on MTP × prefix-caching - # together. Trade vs the alternative (MTP off): this keeps MTP's ~2x decode - # and loses only prefix-cache reuse. GATED ON the multi-turn probe passing. - - --no-enable-prefix-caching + # Prefix caching ON (vLLM V1 default). The APC-off MTP mitigation was tried + # 2026-08-17 and did NOT hold — synthetic 7-turn probes passed but the + # operator still saw severe degeneration in real use, so it was reverted. + # See the MTP block below. + - --enable-prefix-caching - --enable-chunked-prefill - --limit-mm-per-prompt - '{"image": 4}' @@ -102,7 +99,14 @@ services: - --enable-auto-tool-choice - --tool-call-parser - qwen3_coder - # MTP RE-ENABLED 2026-08-17, PAIRED WITH --no-enable-prefix-caching above. + # ⚠️ MTP / SPECULATIVE DECODING DISABLED — the verified known-good. + # Re-enable ONLY after real-use multi-turn testing, not a synthetic probe. + # + # History: MTP-off is clean. The APC-off mitigation (keep MTP, disable + # prefix caching) passed synthetic 7-turn probes on 2026-08-17 but the + # operator STILL saw severe degeneration in real use — so a passing probe + # is NOT sufficient evidence; reverted to MTP-off. The synthetic probes + # structurally under-cover real workloads (content distribution, depth). # # ROOT CAUSE (isolated 2026-08-16, operator-confirmed): `qwen3_5_mtp` # speculative decoding CORRUPTS Qwen3.8-27B output in multi-turn @@ -116,20 +120,16 @@ services: # NOT depth-dependent (n=1 and n=2 both corrupt, so num_speculative_tokens=1 # is NOT a fix). # - # THE MITIGATION (this config): the corruption is gated on MTP × prefix- - # caching TOGETHER (#43559/#47194). Disabling prefix caching forces the GDN - # cache into a mode where the buggy align-path is inert, so MTP can stay on. - # Keeps MTP's ~2x decode; costs only prefix-cache reuse. The proper upstream - # fix (#51113) is in vLLM main / v0.27.2rc0 only — not in a stable release - # as of this date, so we do not jump the fleet gateway to an RC. + # The APC-off mitigation (#43559/#47194) was supposed to make the GDN + # align-path bug inert while keeping MTP. It did not survive real use. + # The proper upstream fix (#51113) is in vLLM main / v0.27.2rc0 only — not + # in a stable release — so we do not jump the fleet gateway to an RC. + # Cost of MTP-off: ~half decode tok/s. Accepted as the known-good. # - # ⚠️ GATED ON THE MULTI-TURN PROBE. If the 7-turn varied series - # (services/gen-seat-mixed-quant/bench/) still collapses past ~2k tokens - # with this config, APC-off did not hold on our stack — revert to MTP OFF - # (comment these two lines, restore --enable-prefix-caching) which is the - # verified known-good. - - --speculative-config - - '{"method": "${GEN_SPEC_METHOD:-qwen3_5_mtp}", "num_speculative_tokens": ${GEN_SPEC_TOKENS:-3}}' + # TO RE-ENABLE (only with real-use verification): uncomment the two lines + # below. Do NOT trust a synthetic probe alone — it passed and was wrong. + # - --speculative-config + # - '{"method": "${GEN_SPEC_METHOD:-qwen3_5_mtp}", "num_speculative_tokens": ${GEN_SPEC_TOKENS:-3}}' deploy: resources: reservations: