From 89c83c42713c5e456adca7d9926ec0a48dc9eec6 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 19 Jun 2026 00:49:04 -0700 Subject: [PATCH] feat(qwen3.5-122b): replace mistral-small-4 as `gen` (abliterated NVFP4, text-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 on ana-ml2 GPU 0 (heretic downed): - stacks/qwen3.5-122b/ — vLLM serve via the repo's text-only patch (Qwen3.5 MoE is a multimodal arch but this checkpoint is text-only weights), --reasoning-parser qwen3, GPU 0 pin, :8013; entrypoint+patch mounted from the model dir. - serve-qwen3.5-122b.yaml — displace heretic + serve + verify. - litellm: REMOVED dead mistral-small-4 / -reasoning; added qwen3.5-122-a10b[-reasoning] + aliases qwen-large[-reasoning] + repointed gen[-reasoning] -> qwen (thinking split via chat_template_kwargs.enable_thinking + --reasoning-parser qwen3). Verified live: qwen healthy on :8013; gen / qwen-large / qwen3.5-122-a10b route, and gen-reasoning returns reasoning_content; mistral-small-4 removed. NOTE: Worldtree character backend (was bound to mistral-small-4) is dark until repointed (operator-acknowledged). --- playbooks/serve-qwen3.5-122b.yaml | 72 ++++++++++++++++++++++++++ stacks/litellm/conf/config.yaml | 84 +++++++++++++++++++------------ stacks/qwen3.5-122b/.env.example | 23 +++++++++ stacks/qwen3.5-122b/compose.yaml | 81 +++++++++++++++++++++++++++++ 4 files changed, 229 insertions(+), 31 deletions(-) create mode 100644 playbooks/serve-qwen3.5-122b.yaml create mode 100644 stacks/qwen3.5-122b/.env.example create mode 100644 stacks/qwen3.5-122b/compose.yaml diff --git a/playbooks/serve-qwen3.5-122b.yaml b/playbooks/serve-qwen3.5-122b.yaml new file mode 100644 index 0000000..663c897 --- /dev/null +++ b/playbooks/serve-qwen3.5-122b.yaml @@ -0,0 +1,72 @@ +# Displace mistral-small-4 (heretic) on ana-ml2 GPU 0 and serve +# bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 (text-only) as the new `gen` model +# (operator 2026-06-19). Weights pre-staged at /tank/aimodels/qwen3.5-122b-a10b-nvfp4 +# (incl. the repo's serving/entrypoint.sh + vllm_patches/ that the compose mounts). +# +# ⚠️ Downing mistral-small-4 takes down the Worldtree CHARACTER backend (vision-intact) +# until it's repointed — operator-acknowledged. REVERT = down qwen, up -d the heretic. +# +# scripts/elway ana-ml2 --playbook playbooks/serve-qwen3.5-122b.yaml + +vars: + compose_dir: /opt/docker/compose/qwen3.5-122b + heretic_dir: /opt/docker/compose/mistral-small-4-heretic + model_dir: /tank/aimodels/qwen3.5-122b-a10b-nvfp4 + host_port: "8013" + +steps: + - name: Verify NVFP4 weights + the repo's patch/entrypoint are staged + shell: | + test -f {{ model_dir }}/model.safetensors.index.json \ + && test -f {{ model_dir }}/serving/entrypoint.sh \ + && test -f {{ model_dir }}/vllm_patches/patch_qwen35_moe_text.py + changed_when: "false" + + - name: Ensure vLLM compile-cache dir exists (writable) + shell: mkdir -p {{ model_dir }}/.cache/vllm + creates: "{{ model_dir }}/.cache/vllm" + + - name: Ensure compose dir exists + shell: mkdir -p {{ compose_dir }} + creates: "{{ compose_dir }}" + + - name: Upload compose.yaml + upload: + src: stacks/qwen3.5-122b/compose.yaml + dest: "{{ compose_dir }}/compose.yaml" + mode: "0644" + + - name: Seed .env from template (only if absent) + upload: + src: stacks/qwen3.5-122b/.env.example + dest: "{{ compose_dir }}/.env" + mode: "0644" + when: "[ ! -f {{ compose_dir }}/.env ]" + + - name: Displace — down mistral-small-4-heretic (frees GPU 0; no-op if down) + shell: cd {{ heretic_dir }} && docker compose down + + - name: Bring up qwen3.5-122b + shell: cd {{ compose_dir }} && docker compose up -d + + - name: Wait for vLLM /health (allow ~15 min for patch + NVFP4 MoE load + warmup) + shell: | + for i in $(seq 1 180); do + curl -sf -o /dev/null --max-time 3 http://localhost:{{ host_port }}/health && exit 0 + sleep 5 + done + exit 1 + changed_when: "false" + +verify: + - name: /health returns 200 + shell: curl -sf -o /dev/null http://localhost:{{ host_port }}/health + changed_when: "false" + + - name: served model id is qwen3.5-122-a10b + shell: curl -sf http://localhost:{{ host_port }}/v1/models | grep -q qwen3.5-122-a10b + changed_when: "false" + + - name: container running + shell: docker inspect vllm-qwen35-122b --format '{{.State.Status}}' | grep -q running + changed_when: "false" diff --git a/stacks/litellm/conf/config.yaml b/stacks/litellm/conf/config.yaml index ceb187b..768a1a3 100644 --- a/stacks/litellm/conf/config.yaml +++ b/stacks/litellm/conf/config.yaml @@ -78,51 +78,73 @@ model_list: model_info: mode: chat - # --- Mistral Small 4 (official NVFP4) — creative-writing / general text. 119B - # MoE (6.5B active), vLLM on ana-ml2 GPU 0 (dedicated 96 GB Blackwell), :8010. - # Explicit entry shadows the "*" wildcard. TEXT-ONLY for now — vLLM 0.23.0's - # Mistral multimodal processor crashes at startup (loaded with image/video - # limit 0); vision returns when vLLM patches it. Deployed 2026-06-15. --- - - model_name: mistral-small-4 + # --- Qwen3.5-122B-A10B (abliterated NVFP4, text-only) — the general / `gen` model. + # REPLACED mistral-small-4 on ana-ml2 GPU 0 (operator 2026-06-19; heretic downed, + # mistral entries REMOVED). bjk110 NVFP4 served on :8013 via vLLM + the text-only + # patch, served-name qwen3.5-122-a10b. Thinking split = chat_template_kwargs. + # enable_thinking + --reasoning-parser qwen3 (mirrors qwen3.6-35b-a3b). One upstream + # fanned out under qwen3.5-122-a10b[-reasoning] + aliases qwen-large[-reasoning] + + # gen[-reasoning]; the -reasoning variants enable thinking. Keep api_base in sync. + # ⚠️ Worldtree CHARACTER backend (was bound to mistral-small-4) is dark until + # repointed — operator-acknowledged. --- + - model_name: qwen3.5-122-a10b litellm_params: - model: hosted_vllm/mistral-small-4 - api_base: http://10.250.50.54:8010/v1 - api_key: os.environ/VLLM_API_KEY - model_info: - mode: chat - - # mistral-small-4-reasoning: same upstream checkpoint, reasoning ON (2026-06-15, - # operator wanted "medium" — but Mistral's reasoning_effort is BINARY, only - # 'none' or 'high' (a medium/low request 400s). 'high' is the sole reasoning-ON - # level, so it carries the -reasoning intent. LiteLLM forwards extra_body to - # vLLM; the mistral reasoning-parser splits [THINK]…[/THINK] into reasoning_content. - - model_name: mistral-small-4-reasoning - litellm_params: - model: hosted_vllm/mistral-small-4 - api_base: http://10.250.50.54:8010/v1 + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY extra_body: - reasoning_effort: high + chat_template_kwargs: + enable_thinking: false + model_info: + mode: chat + - model_name: qwen3.5-122-a10b-reasoning + litellm_params: + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 + api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: true + model_info: + mode: chat + - model_name: qwen-large + litellm_params: + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 + api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: false + model_info: + mode: chat + - model_name: qwen-large-reasoning + litellm_params: + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 + api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: true model_info: mode: chat - - # aliases: gen -> mistral-small-4, gen-reasoning -> mistral-small-4-reasoning - # (operator 2026-06-19). Duplicate-entry aliases — keep api_base in sync with the - # two mistral-small-4 entries above. - model_name: gen litellm_params: - model: hosted_vllm/mistral-small-4 - api_base: http://10.250.50.54:8010/v1 + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY + extra_body: + chat_template_kwargs: + enable_thinking: false model_info: mode: chat - model_name: gen-reasoning litellm_params: - model: hosted_vllm/mistral-small-4 - api_base: http://10.250.50.54:8010/v1 + model: hosted_vllm/qwen3.5-122-a10b + api_base: http://10.250.50.54:8013/v1 api_key: os.environ/VLLM_API_KEY extra_body: - reasoning_effort: high + chat_template_kwargs: + enable_thinking: true model_info: mode: chat diff --git a/stacks/qwen3.5-122b/.env.example b/stacks/qwen3.5-122b/.env.example new file mode 100644 index 0000000..6ed557d --- /dev/null +++ b/stacks/qwen3.5-122b/.env.example @@ -0,0 +1,23 @@ +# qwen3.5-122b (bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4, text-only) — ana-ml2 +# GPU 0 tunables. Real .env lives at /opt/docker/compose/qwen3.5-122b/.env. + +# vllm/vllm-openai:latest per the repo (the text-only patch targets latest). +# MUTABLE tag — pin a digest once a known-good version is established. +QWEN35_IMAGE=vllm/vllm-openai:latest +QWEN35_CONTAINER_NAME=vllm-qwen35-122b + +# Own port (8010=mistral heretic [downed], 8007=qwen36, 8011=selene — 8013 free). +QWEN35_PORT=8013 +QWEN35_GPU_ID=0 + +# vLLM served-model-name; litellm fans out qwen3.5-122-a10b[-reasoning] + aliases +# (qwen-large, gen) onto this, differentiated by chat_template_kwargs.enable_thinking. +QWEN35_SERVED_NAME=qwen3.5-122-a10b + +QWEN35_MAX_MODEL_LEN=131072 +QWEN35_MAX_NUM_SEQS=4 +QWEN35_GPU_MEM_UTIL=0.90 +QWEN35_MAX_NUM_BATCHED_TOKENS=32768 + +# Optional upstream vLLM API key (empty = no auth; internal net only). +API_KEY= diff --git a/stacks/qwen3.5-122b/compose.yaml b/stacks/qwen3.5-122b/compose.yaml new file mode 100644 index 0000000..4838721 --- /dev/null +++ b/stacks/qwen3.5-122b/compose.yaml @@ -0,0 +1,81 @@ +# qwen3.5-122b — bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 on ana-ml2 GPU 0, +# REPLACING mistral-small-4 (operator 2026-06-19: down the heretic, serve this as +# the new general/`gen` model). Abliterated Qwen3.5 MoE (256 experts, 10B active), +# NVFP4 (compressed-tensors), HF format. +# +# SERVING (per the repo's serving/): Qwen3.5 MoE is a MULTIMODAL arch but this +# checkpoint is text-only, so vLLM needs the repo's text-only PATCH applied before +# startup. We reuse the repo's entrypoint.sh (applies the patch, then runs vLLM) and +# vllm_patches/, mounted from the downloaded model dir — keeps patch+checkpoint +# version-coupled. Thinking split via litellm extra_body chat_template_kwargs +# (enable_thinking) + --reasoning-parser qwen3 (mirrors the qwen3.6-35b-a3b pattern). +# +# DISPLACEMENT: GPU 0 fits one mistral-class model; bring this up only after downing +# mistral-small-4-heretic. REVERT = down this, `up -d` the heretic stack. +# +# NOTE: image is vllm/vllm-openai:latest per the repo (the patch targets latest) — +# MUTABLE tag; pin a digest once a known-good version is established. +# +# Tunables in .env. + +name: qwen3.5-122b + +services: + vllm-qwen35-122b: + image: ${QWEN35_IMAGE:-vllm/vllm-openai:latest} + container_name: ${QWEN35_CONTAINER_NAME:-vllm-qwen35-122b} + restart: unless-stopped + ipc: host + ports: + - "${QWEN35_PORT:-8013}:8000" + environment: + - ROLE=head + - TP_SIZE=1 + - MODEL_CONTAINER_PATH=/models/qwen + - SERVED_MODEL_NAME=${QWEN35_SERVED_NAME:-qwen3.5-122-a10b} + - HOST_PORT=8000 + - MAX_MODEL_LEN=${QWEN35_MAX_MODEL_LEN:-131072} + - MAX_NUM_SEQS=${QWEN35_MAX_NUM_SEQS:-4} + - GPU_MEMORY_UTILIZATION=${QWEN35_GPU_MEM_UTIL:-0.90} + - MAX_NUM_BATCHED_TOKENS=${QWEN35_MAX_NUM_BATCHED_TOKENS:-32768} + # --reasoning-parser qwen3 surfaces … as reasoning_content; + # the thinking on/off itself is per-request (litellm chat_template_kwargs). + - VLLM_EXTRA_ARGS=--reasoning-parser qwen3 --enable-chunked-prefill + - NVIDIA_VISIBLE_DEVICES=0 + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + - VLLM_API_KEY=${API_KEY:-} + volumes: + # checkpoint + the repo's entrypoint/patch (downloaded with the model). + - /tank/aimodels/qwen3.5-122b-a10b-nvfp4:/models/qwen:ro + - /tank/aimodels/qwen3.5-122b-a10b-nvfp4/serving/entrypoint.sh:/entrypoint.sh:ro + - /tank/aimodels/qwen3.5-122b-a10b-nvfp4/vllm_patches:/patches:ro + - /tank/aimodels/qwen3.5-122b-a10b-nvfp4/.cache/vllm:/root/.cache/vllm + entrypoint: ["bash", "/entrypoint.sh"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: + - "${QWEN35_GPU_ID:-0}" + capabilities: + - gpu + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + networks: + - tnet + labels: + - homepage.group=AI Systems + - homepage.name=Qwen3.5-122B-A10B (abliterated NVFP4) + - homepage.icon=mdi-creation + - homepage.description=Abliterated Qwen3.5 122B-A10B NVFP4, the new `gen` model (ana-ml2 GPU 0) + - homepage.href=http://10.250.50.54:${QWEN35_PORT:-8013}/docs + +networks: + tnet: + name: traefik-net + external: true