vh
5f049cb4ad
SGLang 0.5.13 confirmed to support our formats on Blackwell sm_120 (compressed-tensors NVFP4 W4A4, fp8, modelopt_fp4, petit_nvfp4, fp4_e2m1 KV), so the bench can be a real NVFP4 head-to-head. Parameterized compose (model/ quant/GPU via .env) + a common streaming load generator (bench.py: agg tok/s, TTFT p50/p99, TPOT) so both engines are driven identically on an exclusive GPU. Bench-oriented; promote to a real stack only if SGLang wins. Launch deferred until the NVFP4 eval frees a GPU.
78 lines
2.6 KiB
YAML
78 lines
2.6 KiB
YAML
# SGLang — alternative LLM serving engine, stood up on ana-ml2 to bench
|
|
# head-to-head against vLLM on the same model + hardware.
|
|
#
|
|
# Bench-oriented (not yet a permanent service): point it at the SAME checkpoint
|
|
# vLLM serves, give it an EXCLUSIVE GPU (fair benches need no co-tenants), and
|
|
# drive both engines with one common load generator. If SGLang wins decisively,
|
|
# promote this to a real stack; otherwise tear it down after the bench.
|
|
#
|
|
# SGLang 0.5.13 supports our formats on Blackwell sm_120: compressed-tensors
|
|
# (the llm-compressor NVFP4 W4A4 output), fp8, modelopt_fp4, petit_nvfp4, and
|
|
# fp4_e2m1 KV. So SGLANG_QUANT can be compressed-tensors (NVFP4) or fp8.
|
|
#
|
|
# All tunables in .env — edit that, not this file.
|
|
|
|
services:
|
|
sglang:
|
|
image: lmsysorg/sglang:${SGLANG_VERSION}
|
|
container_name: sglang
|
|
restart: unless-stopped
|
|
ipc: host # SGLang needs large shared memory
|
|
ports:
|
|
- "${SGLANG_PORT}:30000"
|
|
volumes:
|
|
- /tank/aimodels:/aimodels:ro # NVFP4 checkpoints + HF cache live here
|
|
environment:
|
|
- HF_HOME=/aimodels/huggingface
|
|
- HF_HUB_CACHE=/aimodels/huggingface/hub
|
|
command:
|
|
- python3
|
|
- -m
|
|
- sglang.launch_server
|
|
- --model-path
|
|
- ${SGLANG_MODEL} # e.g. /aimodels/nvfp4/granite-4.1-8b-NVFP4-W4A4-test
|
|
- --served-model-name
|
|
- ${SGLANG_SERVED_NAME}
|
|
- --host
|
|
- 0.0.0.0
|
|
- --port
|
|
- "30000"
|
|
- --quantization
|
|
- ${SGLANG_QUANT} # compressed-tensors (NVFP4) | fp8
|
|
- --kv-cache-dtype
|
|
- ${SGLANG_KV_DTYPE:-fp8_e4m3}
|
|
- --context-length
|
|
- ${SGLANG_CTX_LEN:-65536}
|
|
- --mem-fraction-static
|
|
- ${SGLANG_MEM_FRACTION:-0.85} # SGLang's analog of vLLM gpu-memory-utilization
|
|
- --tp
|
|
- "1"
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
device_ids:
|
|
- "${SGLANG_GPU_ID}" # exclusive GPU for a fair bench
|
|
capabilities:
|
|
- gpu
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "python3 -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:30000/health').status==200 else 1)\""]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
start_period: 300s
|
|
networks:
|
|
- tnet
|
|
labels:
|
|
- homepage.group=AI Systems
|
|
- homepage.name=SGLang (bench)
|
|
- homepage.icon=mdi-speedometer
|
|
- homepage.description=SGLang serving — vLLM bench comparison (ana-ml2)
|
|
- homepage.href=http://10.250.50.54:${SGLANG_PORT}
|
|
|
|
networks:
|
|
tnet:
|
|
name: traefik-net
|
|
external: true
|