# SGLang — alternative LLM serving engine, stood up on fv-ml1 to bench
# head-to-head against vLLM on the same model + hardware.
#
# Bench-oriented (not yet a permanent service): point it at the SAME checkpoint
# vLLM serves, give it an EXCLUSIVE GPU (fair benches need no co-tenants), and
# drive both engines with one common load generator. If SGLang wins decisively,
# promote this to a real stack; otherwise tear it down after the bench.
#
# SGLang 0.5.13 supports our formats on Blackwell sm_120: compressed-tensors
# (the llm-compressor NVFP4 W4A4 output), fp8, modelopt_fp4, petit_nvfp4, and
# fp4_e2m1 KV. So SGLANG_QUANT can be compressed-tensors (NVFP4) or fp8.
#
# All tunables in .env — edit that, not this file.

services:
  sglang:
    image: lmsysorg/sglang:${SGLANG_VERSION}
    container_name: sglang
    restart: unless-stopped
    ipc: host                      # SGLang needs large shared memory
    ports:
      - "${SGLANG_PORT}:30000"
    volumes:
      - /tank/aimodels:/aimodels:ro     # NVFP4 checkpoints + HF cache live here
    environment:
      - HF_HOME=/aimodels/huggingface
      - HF_HUB_CACHE=/aimodels/huggingface/hub
    command:
      - python3
      - -m
      - sglang.launch_server
      - --model-path
      - ${SGLANG_MODEL}                 # e.g. /aimodels/nvfp4/granite-4.1-8b-NVFP4-W4A4-test
      - --served-model-name
      - ${SGLANG_SERVED_NAME}
      - --host
      - 0.0.0.0
      - --port
      - "30000"
      - --quantization
      - ${SGLANG_QUANT}                 # compressed-tensors (NVFP4) | fp8
      - --kv-cache-dtype
      - ${SGLANG_KV_DTYPE:-fp8_e4m3}
      - --context-length
      - ${SGLANG_CTX_LEN:-65536}
      - --mem-fraction-static
      - ${SGLANG_MEM_FRACTION:-0.85}     # SGLang's analog of vLLM gpu-memory-utilization
      - --tp
      - "1"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids:
                - "${SGLANG_GPU_ID}"    # exclusive GPU for a fair bench
              capabilities:
                - gpu
    healthcheck:
      test: ["CMD-SHELL", "python3 -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:30000/health').status==200 else 1)\""]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 300s
    networks:
      - tnet
    labels:
      - homepage.group=AI Systems
      - homepage.name=SGLang (bench)
      - homepage.icon=mdi-speedometer
      - homepage.description=SGLang serving — vLLM bench comparison (fv-ml1)
      - homepage.href=http://10.251.50.54:${SGLANG_PORT}

networks:
  tnet:
    name: traefik-net
    external: true