gen-seat: deploy Qwen3.8-27B-Uncensored gen seat, rename qwen36-27b-aeon->gen-seat
- New uncensored gen seat: JonathanColetti/Qwen3.8-27B-Uncensored, in-house NVFP4 W4A16 (compressed-tensors) + grafted bf16 MTP (config ignore re:^mtp.*), vision-intact, 262K ctx, MTP n=3 (~42% accept, ~68 tok/s). Replaces the qwen3.6-35b-a3b-heretic MoE. - Rename compose project qwen36-27b-aeon -> gen-seat, container vllm-aeon-gen -> vllm-gen, env vars AEON_GEN_* -> GEN_*; drop the dormant vllm-aeon-rp service. - litellm: repoint 7 aliases (gen/summarizer/summarizer-large/classifier/image-judge/ qwen-image-bench -> qwen3.8-27b-uncensored; gen-reasoning -> -thinking). - servers/ana-ml2/README: refresh the gen hero-seat row.
This commit is contained in:
@@ -22,44 +22,51 @@ model_list:
|
||||
# R15 P03 eval). vLLM on ana-ml2 GPU 1, official FP8, 50K ctx. Explicit
|
||||
# entry shadows the "*" wildcard's llama-swap route for this name. Full
|
||||
# prompt + completion captured per call. ---
|
||||
- model_name: granite-4.1-8b
|
||||
litellm_params:
|
||||
model: hosted_vllm/granite-4.1-8b
|
||||
api_base: http://10.250.50.54:8004/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
temperature: 0
|
||||
model_info:
|
||||
mode: chat
|
||||
# --- granite-4.1-8b RETIRED 2026-08-12 (seat downed, GPU1 reclaimed for RP context) ---
|
||||
# - model_name: granite-4.1-8b
|
||||
# litellm_params:
|
||||
# model: hosted_vllm/granite-4.1-8b
|
||||
# api_base: http://10.250.50.54:8004/v1
|
||||
# api_key: os.environ/VLLM_API_KEY
|
||||
# temperature: 0
|
||||
# model_info:
|
||||
# mode: chat
|
||||
|
||||
# alias: summarizer -> granite-4.1-8b (operator 2026-06-19). Duplicate-entry alias
|
||||
# alias: summarizer -> gen (repointed 2026-08-12, granite retired) (operator 2026-06-19). Duplicate-entry alias
|
||||
# (not router_settings.model_group_alias — that's hidden from /v1/models and can be
|
||||
# silently ignored in config per litellm #15020/#5524). Keep api_base in sync above.
|
||||
- model_name: summarizer
|
||||
litellm_params:
|
||||
model: hosted_vllm/granite-4.1-8b
|
||||
api_base: http://10.250.50.54:8004/v1
|
||||
model: hosted_vllm/qwen3.8-27b-uncensored
|
||||
api_base: http://10.250.50.54:8015/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
temperature: 0
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: false
|
||||
model_info:
|
||||
mode: chat
|
||||
|
||||
# alias: classifier -> granite-4.1-8b (operator 2026-06-19). Light/fast classification
|
||||
# alias: classifier -> gen (repointed 2026-08-12, granite retired) (operator 2026-06-19). Light/fast classification
|
||||
# + triage endpoint; same backend as summarizer. Keep api_base in sync above.
|
||||
- model_name: classifier
|
||||
litellm_params:
|
||||
model: hosted_vllm/granite-4.1-8b
|
||||
api_base: http://10.250.50.54:8004/v1
|
||||
model: hosted_vllm/qwen3.8-27b-uncensored
|
||||
api_base: http://10.250.50.54:8015/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
temperature: 0
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: false
|
||||
model_info:
|
||||
mode: chat
|
||||
|
||||
# alias: summarizer-large -> gen / qwen3.6-35b-a3b-heretic (operator 2026-07-05). For heavier
|
||||
# alias: summarizer-large -> gen / qwen3.8-27b-uncensored (operator 2026-07-05). For heavier
|
||||
# summarization that wants the 35B-A3B heretic `gen` model instead of granite-8b. Thinking OFF
|
||||
# (matches gen). Keep api_base (:8015) + enable_thinking in sync with the gen record below.
|
||||
- model_name: summarizer-large
|
||||
litellm_params:
|
||||
model: hosted_vllm/qwen3.6-35b-a3b-heretic
|
||||
model: hosted_vllm/qwen3.8-27b-uncensored
|
||||
api_base: http://10.250.50.54:8015/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
presence_penalty: 1.5
|
||||
@@ -75,14 +82,14 @@ model_list:
|
||||
# --- image-judge / qwen-image-bench — T2I quality JUDGE aliases. The dedicated
|
||||
# Qwen-Image-Bench NVFP4 backend (ana-ml2 GPU 1, :8014) was RETIRED 2026-07-15
|
||||
# (operator: reclaim ~30GB GPU1) after the arbo→gen hero-judge switch. Both
|
||||
# aliases now REPOINT to the gen backend (:8015, qwen3.6-35b-a3b-heretic,
|
||||
# aliases now REPOINT to the gen backend (:8015, qwen3.8-27b-uncensored,
|
||||
# vision-intact), held at deterministic judge sampling (temp 0 / top_k 1) with
|
||||
# enable_thinking:false (a reasoning preamble breaks json_object). Revert =
|
||||
# `docker compose start` stacks/qwen-image-bench on ana-ml2 + repoint api_base
|
||||
# back to :8014 + model hosted_vllm/qwen-image-bench. ---
|
||||
- model_name: qwen-image-bench
|
||||
litellm_params:
|
||||
model: hosted_vllm/qwen3.6-35b-a3b-heretic
|
||||
model: hosted_vllm/qwen3.8-27b-uncensored
|
||||
api_base: http://10.250.50.54:8015/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
temperature: 0
|
||||
@@ -96,7 +103,7 @@ model_list:
|
||||
mode: chat
|
||||
- model_name: image-judge
|
||||
litellm_params:
|
||||
model: hosted_vllm/qwen3.6-35b-a3b-heretic
|
||||
model: hosted_vllm/qwen3.8-27b-uncensored
|
||||
api_base: http://10.250.50.54:8015/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
temperature: 0
|
||||
@@ -114,7 +121,7 @@ model_list:
|
||||
# Displaced AEON-27B 2026-07-08 (which had displaced qwopus3.5-122b 2026-07-05). MoE 35B-A3B
|
||||
# (256 experts / 8 active), qwen3_5_moe GDN-hybrid, native MTP preserved but served MTP-OFF
|
||||
# (spec-decode hurts concurrent aggregate). Served on :8015 via vLLM,
|
||||
# served-name qwen3.6-35b-a3b-heretic. Thinking split = chat_template_kwargs.enable_thinking +
|
||||
# served-name qwen3.8-27b-uncensored. Thinking split = chat_template_kwargs.enable_thinking +
|
||||
# --reasoning-parser qwen3; tool-calling qwen3_coder. gen / gen-reasoning + summarizer-
|
||||
# large route here; -reasoning enables thinking. Keep api_base (:8015) in sync.
|
||||
# RETIRED with the displacement (→ 404, callers migrate to gen): qwen3.5-122-a10b
|
||||
@@ -126,7 +133,7 @@ model_list:
|
||||
# GENERAL thinking profile, not the 0.6 coding sub-profile). docs/pfi/model-sampler-defaults.md. ---
|
||||
- model_name: gen
|
||||
litellm_params:
|
||||
model: hosted_vllm/qwen3.6-35b-a3b-heretic
|
||||
model: hosted_vllm/qwen3.8-27b-uncensored
|
||||
api_base: http://10.250.50.54:8015/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
presence_penalty: 1.5
|
||||
@@ -142,7 +149,7 @@ model_list:
|
||||
litellm_params:
|
||||
# Distinct served-name so a thinking-off `gen` request can't mutate this deployment's
|
||||
# enable_thinking (shared-config-mutation footgun). Same backend :8015, different model id.
|
||||
model: hosted_vllm/qwen3.6-35b-a3b-heretic-thinking
|
||||
model: hosted_vllm/qwen3.8-27b-uncensored-thinking
|
||||
api_base: http://10.250.50.54:8015/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
presence_penalty: 1.5
|
||||
@@ -167,7 +174,7 @@ model_list:
|
||||
# docs/pfi/model-sampler-defaults.md.
|
||||
- model_name: char-rp
|
||||
litellm_params:
|
||||
model: hosted_vllm/magidonia-24b-v4.3
|
||||
model: hosted_vllm/char-rp
|
||||
api_base: http://10.250.50.54:8016/v1
|
||||
api_key: os.environ/VLLM_API_KEY
|
||||
temperature: 1.1
|
||||
@@ -248,9 +255,7 @@ model_list:
|
||||
top_p: 0.9
|
||||
model_info:
|
||||
mode: chat
|
||||
# reranker → generic capability name for rerank. Repointed to bge-reranker-v2-m3
|
||||
# 2026-08-06 (Brokkr R43: incumbent Qwen3-Reranker-0.6B harmed 80/90 fleet queries;
|
||||
# boundary 17:37:48Z). Incumbent stays reachable via the qwen3-reranker alias for rollback.
|
||||
# reranker → generic capability name for rerank (currently qwen3-reranker).
|
||||
- model_name: reranker
|
||||
litellm_params:
|
||||
model: hosted_vllm/BAAI/bge-reranker-v2-m3
|
||||
|
||||
Reference in New Issue
Block a user