asset-engine: scaffold deploy stack on ana-docker

Mirrors task-board's build-on-host pattern: elway playbook clones
vh/asset-engine into /opt/docker/build/, docker build, install compose +
seed .env, up -d, verify /health. No registry.

Internal-only tool — LAN port 8200 (bind 0.0.0.0) is primary; Traefik
labels additionally route asset-engine.phasefinal.com with TLS via the
anaprod cert resolver. DB and outputs are separate bind-mounts under
/opt/docker/conf/asset-engine/ so outputs/ can move volumes later
without touching DB state. INFERENCE_HOST defaults to 10.100.79.3
(irv-ml1 over WG). OIDC env seam is pre-allocated empty for v2.

playbooks/deploy-asset-engine.yaml

@@ -0,0 +1,136 @@
+# Deploy asset-engine (https://gitea.phasefinal.com/vh/asset-engine) to a
+# Docker host following the PFI /opt/docker/ convention (ana-docker by
+# default, but the playbook works against any host with Docker +
+# traefik-net in place).
+#
+# Idempotent: rerunning is safe. Creates-gates + conditional when:
+# checks skip work that's already done; `docker compose up -d` is itself
+# idempotent (no restart unless compose content or env changed).
+#
+# Usage:
+#   scripts/elway ana-docker --playbook playbooks/deploy-asset-engine.yaml
+#   scripts/elway ana-docker --playbook playbooks/deploy-asset-engine.yaml --var ref=v0.1.0
+#
+# Prereqs on the target host:
+#   - Docker + docker compose plugin
+#   - `traefik-net` docker network (external)
+#   - Target user (lkraven) has git SSH access to gitea.phasefinal.com
+#     — either SSH key authorized in gitea, or the repo is HTTPS-reachable
+#     if you swap `repo_url` below.
+#   - Target user is in the `docker` group.
+
+vars:
+  repo_url: git@gitea.phasefinal.com:vh/asset-engine.git
+  ref: main
+  build_dir: /opt/docker/build/asset-engine
+  image_tag: asset-engine:local
+  compose_dir: /opt/docker/compose/asset-engine
+  db_dir: /opt/docker/conf/asset-engine/db
+  outputs_dir: /opt/docker/conf/asset-engine/outputs
+  host_port: "8200"
+
+steps:
+  # ── host-side directory prep ─────────────────────────────────────────
+  - name: Ensure /opt/docker/build parent exists
+    shell: mkdir -p /opt/docker/build
+    sudo: true
+    creates: /opt/docker/build
+
+  - name: Chown /opt/docker/build to lkraven (only if mkdir'd by root above)
+    shell: chown lkraven:lkraven /opt/docker/build
+    sudo: true
+    when: '[ "$(stat -c %U /opt/docker/build)" != lkraven ]'
+
+  # ── fetch / sync source ─────────────────────────────────────────────
+  - name: Clone asset-engine repo if absent
+    # Auto-accept the first-run host key so the playbook doesn't hang
+    # prompting for yes/no.
+    shell: GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new" git clone {{ repo_url }} {{ build_dir }}
+    creates: "{{ build_dir }}/.git"
+
+  - name: Fetch from origin
+    shell: cd {{ build_dir }} && git fetch --quiet origin
+
+  - name: Reset working tree to {{ ref }}
+    # Accept either a branch name (resolves via origin/<ref>) or a
+    # full/short SHA (resolves directly). CI passes the triggering
+    # commit SHA via --var ref=${{ github.sha }}; manual runs pass
+    # branch names like main / v0.1.0.
+    shell: |
+      cd {{ build_dir }}
+      if sha=$(git rev-parse --verify --quiet "origin/{{ ref }}^{commit}"); then :;
+      elif sha=$(git rev-parse --verify --quiet "{{ ref }}^{commit}"); then :;
+      else echo "elway: ref not found: {{ ref }}" >&2; exit 1; fi
+      git reset --hard "$sha"
+    # Report ok (no-change) when the tree was already at the requested
+    # ref — saves a noisy CHANGED status line on no-op reruns.
+    changed_when: '[ "$(cd {{ build_dir }} && git rev-parse HEAD)" != "$(cd {{ build_dir }} && (git rev-parse --verify --quiet "origin/{{ ref }}^{commit}" || git rev-parse --verify --quiet "{{ ref }}^{commit}"))" ]'
+
+  # ── image build ─────────────────────────────────────────────────────
+  - name: Build image {{ image_tag }}
+    shell: cd {{ build_dir }} && docker build -t {{ image_tag }} .
+    # Docker build reuses layer cache and is fast on reruns, but it
+    # always runs — we can't cheaply know up-front whether anything
+    # downstream has changed. Leave it in the always-run lane; Docker
+    # itself handles the no-op efficiently.
+
+  # ── compose + state dirs ────────────────────────────────────────────
+  - name: Ensure compose dir exists
+    shell: mkdir -p {{ compose_dir }}
+    creates: "{{ compose_dir }}"
+
+  - name: Ensure DB dir exists
+    # Created as lkraven (uid 1000 on these hosts), matching the
+    # container's app user — no chown dance needed.
+    shell: mkdir -p {{ db_dir }}
+    creates: "{{ db_dir }}"
+
+  - name: Ensure outputs dir exists
+    # Separate from db_dir so outputs/ can later move to a bigger
+    # volume without touching DB state.
+    shell: mkdir -p {{ outputs_dir }}
+    creates: "{{ outputs_dir }}"
+
+  # ── deploy compose files ────────────────────────────────────────────
+  - name: Upload compose.yaml
+    upload:
+      src: stacks/asset-engine/compose.yaml
+      dest: "{{ compose_dir }}/compose.yaml"
+      mode: "0644"
+
+  - name: Seed .env from template (only if absent)
+    upload:
+      src: stacks/asset-engine/.env.example
+      dest: "{{ compose_dir }}/.env"
+      mode: "0644"
+    when: "[ ! -f {{ compose_dir }}/.env ]"
+
+  # ── bring up + wait for ready ───────────────────────────────────────
+  - name: docker compose up -d
+    shell: cd {{ compose_dir }} && docker compose up -d
+
+  - name: Wait for /health to respond
+    # Short retry loop — docker compose up returns before healthcheck
+    # stabilizes; we want verify: to run against a live server.
+    shell: |
+      for i in $(seq 1 30); do
+        curl -sf -o /dev/null http://localhost:{{ host_port }}/health && exit 0
+        sleep 1
+      done
+      exit 1
+    changed_when: "false"
+
+verify:
+  - name: /health returns 200
+    shell: curl -sf -o /dev/null http://localhost:{{ host_port }}/health
+    changed_when: "false"
+
+  - name: /health body reports status=ok
+    shell: curl -sf http://localhost:{{ host_port }}/health | grep -q '"status"[[:space:]]*:[[:space:]]*"ok"'
+    changed_when: "false"
+
+  - name: Container is in the traefik-net network
+    # `traefik-net` has a dash, so it's not accessible via Go template dot
+    # syntax — JSON-encode the networks map and grep for the key instead.
+    shell: docker inspect asset-engine --format '{{json .NetworkSettings.Networks}}' | grep -q traefik-net
+    changed_when: "false"

stacks/asset-engine/.env.example

@@ -0,0 +1,37 @@
+# asset-engine stack tunables. Copy to `.env` on ana-docker before deploying.
+#
+# The deploy playbook seeds `.env` from this template on first run only —
+# it won't clobber an existing `.env`.
+
+# Image tag. Built locally from the asset-engine git repo by the playbook.
+ASSET_ENGINE_IMAGE=asset-engine:local
+
+# Host port exposing the FastAPI app (container listens on 8000 internally).
+# Internal-only tool — LAN access on this port is the primary entry; the
+# Traefik labels in compose.yaml additionally route the public hostname.
+ASSET_ENGINE_PORT=8200
+
+# Bind address for the host port. 0.0.0.0 = LAN-reachable (default for an
+# internal-only tool). Flip to 127.0.0.1 only if you want to force all
+# traffic through traefik.
+ASSET_ENGINE_BIND=0.0.0.0
+
+# Host paths for state. Container runs as uid 1000 — paths must be writable
+# by that uid (mkdir'd by the playbook without sudo, so lkraven-owned when
+# lkraven is uid 1000 on the host).
+#
+# DB lives separately from outputs so we can grow outputs/ onto a different
+# volume later without restoring DB state on top of it.
+ASSET_ENGINE_DB_DIR=/opt/docker/conf/asset-engine/db
+ASSET_ENGINE_OUTPUTS_DIR=/opt/docker/conf/asset-engine/outputs
+
+# Inference target. Default is irv-ml1 over WG. Override if the fleet's
+# inference host moves.
+INFERENCE_HOST=10.100.79.3
+
+# OIDC seam — empty in v1 (auth is no-op). Populate when v2 forward-auth
+# lands. Pre-allocated here so the surface is visible in the config file
+# before code reads it.
+OIDC_ISSUER=
+OIDC_CLIENT_ID=
+OIDC_CLIENT_SECRET=

stacks/asset-engine/README.md

@@ -0,0 +1,85 @@
+# asset-engine
+
+Control plane over the PFI inference fleet — FastAPI + HTMX/Shoelace
+UI that exposes the catalog at [`docs/asset-engine/services.yaml`](../../docs/asset-engine/services.yaml)
+as a web app, routes generation requests to inference hosts (irv-ml1
+over WG by default), and persists generated assets to a local SQLite
+DB + content-addressed blob store.
+
+**Server:** ana-docker
+**Hostname (TLS):** `asset-engine.phasefinal.com` (TLS via Traefik / `anaprod` cert resolver)
+**LAN port:** `10.250.50.70:8200` (configurable via `.env`)
+**Upstream repo:** [vh/asset-engine](https://gitea.phasefinal.com/vh/asset-engine)
+**Image:** `asset-engine:local` — built on the host from the git repo by
+the deploy playbook. Not pulled from a registry.
+
+## Deploy
+
+Two paths — automated (preferred) and manual (escape hatch / first-time).
+
+### Automated (Gitea Actions, push-to-main)
+
+The asset-engine repo ships `.gitea/workflows/{ci,deploy}.yaml`. CI runs
+on PRs (uv sync, pytest, catalog drift check against this repo's
+`docs/asset-engine/services.yaml`); the deploy workflow runs on push to
+main and just calls the elway playbook below pinned to the triggering
+commit SHA. Drift check is a BLOCKING gate — a PR that vendors a
+services.yaml mismatched against this repo fails CI and can't merge.
+
+A reference copy of the deploy workflow lives next to this README at
+[`gitea-workflow-deploy.yaml.example`](gitea-workflow-deploy.yaml.example);
+the canonical source is in the asset-engine repo. The example header
+lists the two repo secrets required (`DEPLOY_SSH_KEY`, `MGMT_REPO_TOKEN`).
+
+### Manual (elway from a workstation)
+
+The playbook owns the full flow: clone/update the source repo,
+`docker build`, install compose + seed .env, bring up, verify health.
+
+```bash
+# First deploy (or update to latest main)
+scripts/elway ana-docker --playbook playbooks/deploy-asset-engine.yaml
+
+# Pin to a specific ref (tag, branch, or commit SHA)
+scripts/elway ana-docker --playbook playbooks/deploy-asset-engine.yaml --var ref=v0.1.0
+```
+
+## Path layout (on ana-docker)
+
+| Host path | Container path | Purpose | Restic? |
+|---|---|---|---|
+| `/opt/docker/build/asset-engine/` | — | git checkout used as docker build context | excluded |
+| `/opt/docker/compose/asset-engine/` | — | compose.yaml + .env | included (via `/opt/docker`) |
+| `/opt/docker/conf/asset-engine/db/` | `/app/runtime/db` | SQLite (`asset_engine.db` + WAL) | **included** |
+| `/opt/docker/conf/asset-engine/outputs/` | `/app/runtime/outputs` | content-addressed blob store | **included** |
+
+## Network model
+
+Internal-only tool, two entry points:
+
+- **LAN**, default: `http://10.250.50.70:8200` — container port 8000
+  published on the host, bound to 0.0.0.0 (configurable via
+  `ASSET_ENGINE_BIND` / `ASSET_ENGINE_PORT`).
+- **TLS hostname**: `https://asset-engine.phasefinal.com` — Traefik on
+  ana-docker terminates TLS and forwards to the container over the
+  `traefik-net` docker network on port 8000.
+
+`INFERENCE_HOST` defaults to `10.100.79.3` (irv-ml1 over WG). Override
+in `.env` if the fleet's inference topology moves.
+
+## Catalog drift
+
+`docs/asset-engine/services.yaml` in this repo is the canonical
+catalog. The asset-engine repo vendors a copy at `data/services.yaml`
+and re-vendors via `uv run scripts/sync_catalog.py` after upstream
+changes; CI fails any PR where the vendored copy diverges from this
+one. Pattern is: edit catalog here → asset-engine re-vendors → both
+sides commit on the same merge window.
+
+## Outputs directory growth
+
+`outputs/` grows unbounded in v1 — `Asset.retention` exists in the
+schema but the GC sweep isn't wired yet. The plan: Beszel alert when
+`du -sh /opt/docker/conf/asset-engine/outputs` crosses ~50 GB,
+revisit the threshold once we have real growth data. Tracking issue
+in the asset-engine repo.

stacks/asset-engine/compose.yaml

@@ -0,0 +1,66 @@
+# asset-engine — control plane over PFI's inference fleet.
+#
+# FastAPI + HTMX/Shoelace UI. Reads the catalog at services.yaml (baked
+# into the image at build time, drift-checked in CI against the copy
+# at docs/asset-engine/services.yaml in this repo), routes requests to
+# inference hosts (irv-ml1 over WG by default), persists Assets to a
+# local SQLite WAL DB, stores generated blobs content-addressed under
+# runtime/outputs/.
+#
+# Image is built on the host from the asset-engine git repo by the
+# deploy playbook (`playbooks/deploy-asset-engine.yaml`), which clones
+# into /opt/docker/build/asset-engine and runs `docker build -t
+# asset-engine:local .` before installing this compose and bringing
+# it up. No registry.
+#
+# State persists under /opt/docker/conf/asset-engine/{db,outputs} on
+# the host — separate bind-mounts so outputs/ can move to a bigger
+# volume later without touching DB state.
+#
+# All tunables live in .env — edit that, not this file.
+
+services:
+  asset-engine:
+    image: ${ASSET_ENGINE_IMAGE}
+    container_name: asset-engine
+    restart: unless-stopped
+    ports:
+      - "${ASSET_ENGINE_BIND:-0.0.0.0}:${ASSET_ENGINE_PORT}:8000"
+    environment:
+      # App always listens on 8000 internally; host port is the only knob.
+      - RUNTIME_DIR=/app/runtime
+      - DB_URL=sqlite:////app/runtime/db/asset_engine.db
+      - INFERENCE_HOST=${INFERENCE_HOST:-10.100.79.3}
+      # OIDC seam is empty in v1; populated in v2 when forward-auth lands.
+      - OIDC_ISSUER=${OIDC_ISSUER:-}
+      - OIDC_CLIENT_ID=${OIDC_CLIENT_ID:-}
+      - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET:-}
+    volumes:
+      - ${ASSET_ENGINE_DB_DIR}:/app/runtime/db
+      - ${ASSET_ENGINE_OUTPUTS_DIR}:/app/runtime/outputs
+    healthcheck:
+      test: ["CMD-SHELL", "python -c 'import urllib.request,sys; r=urllib.request.urlopen(\"http://127.0.0.1:8000/health\",timeout=3); sys.exit(0 if r.status==200 else 1)' || exit 1"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 30s
+    networks:
+      - tnet
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.asset-engine.rule=Host(`asset-engine.phasefinal.com`)
+      - traefik.http.routers.asset-engine.entrypoints=websecure
+      - traefik.http.routers.asset-engine.tls=true
+      - traefik.http.routers.asset-engine.tls.certresolver=anaprod
+      - traefik.http.routers.asset-engine.middlewares=crowdsec@file
+      - traefik.http.services.asset-engine.loadbalancer.server.port=8000
+      - homepage.group=AI Systems
+      - homepage.name=Asset Engine
+      - homepage.icon=mdi-tools
+      - homepage.description=Control plane over the PFI inference fleet
+      - homepage.href=https://asset-engine.phasefinal.com
+
+networks:
+  tnet:
+    name: traefik-net
+    external: true

stacks/asset-engine/gitea-workflow-deploy.yaml.example

@@ -0,0 +1,91 @@
+# Gitea Actions workflow for asset-engine.
+#
+# THIS FILE LIVES IN THE ASSET-ENGINE REPO, NOT HERE.
+# Copy to vh/asset-engine:.gitea/workflows/deploy.yaml and commit.
+# (The canonical copy lives in the asset-engine repo; this file is a
+# reference for what shape the workflow takes.)
+#
+# What it does on every push to main (and on manual workflow_dispatch):
+#   1. Checks out asset-engine itself (the triggering repo).
+#   2. Checks out vh/esh-pfi-infrastructure to pick up the elway
+#      playbook and helper scripts.
+#   3. Configures SSH so elway can reach ana-docker.
+#   4. Runs `scripts/elway ana-docker --playbook playbooks/deploy-asset-engine.yaml`
+#      pinning to the commit SHA that triggered the workflow.
+#
+# Required Actions secrets (configure under
+# https://gitea.phasefinal.com/vh/asset-engine/settings/actions/secrets,
+# or org-level for reuse across repos):
+#
+#   DEPLOY_SSH_KEY    Private SSH key whose pubkey is in
+#                     ~lkraven/.ssh/authorized_keys on ana-docker.
+#                     Used by the runner to invoke the elway playbook.
+#                     Generate fresh; don't reuse a personal key.
+#
+#   MGMT_REPO_TOKEN   Gitea PAT (read:repository scope) on
+#                     vh/esh-pfi-infrastructure, used to clone the
+#                     management repo. Generate at
+#                     https://gitea.phasefinal.com/-/user/settings/applications.
+
+name: Deploy asset-engine
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    # `pfi-fleet` matches the central runner on ana-docker. Pin to
+    # `ana-docker` instead if you want to refuse running on a future
+    # site-local runner. The runner's label embeds a default image
+    # (node:20-bookworm-slim) — has node + git out of the box, so
+    # actions/checkout@v4 (a JS action) works without a custom
+    # container. We just apt-install python3 + pyyaml for elway.
+    runs-on: pfi-fleet
+
+    steps:
+      - name: Install playbook prerequisites
+        run: |
+          apt-get update -qq
+          apt-get install -y --no-install-recommends \
+            python3 python3-yaml openssh-client
+          rm -rf /var/lib/apt/lists/*
+
+      - name: Checkout asset-engine (triggering repo)
+        uses: actions/checkout@v4
+
+      - name: Checkout management repo (eshpfi-management)
+        uses: actions/checkout@v4
+        with:
+          repository: vh/esh-pfi-infrastructure
+          token: ${{ secrets.MGMT_REPO_TOKEN }}
+          path: _mgmt
+
+      - name: Configure SSH to ana-docker
+        run: |
+          mkdir -p ~/.ssh
+          # The DEPLOY_SSH_KEY secret is the full private key contents,
+          # newline-terminated. ssh refuses keys that aren't 0600.
+          printf '%s\n' "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+
+          # ssh_config alias so elway resolves "ana-docker" the same
+          # way it would on a workstation. accept-new is fine for a
+          # fresh job container — host key gets cached for the lifetime
+          # of this job only.
+          cat > ~/.ssh/config <<'EOF'
+          Host ana-docker
+            HostName 10.250.50.70
+            User lkraven
+            IdentityFile ~/.ssh/id_ed25519
+            StrictHostKeyChecking accept-new
+          EOF
+          chmod 600 ~/.ssh/config
+
+      - name: Deploy asset-engine (elway playbook, pinned to this commit)
+        working-directory: _mgmt
+        run: |
+          scripts/elway ana-docker \
+            --playbook playbooks/deploy-asset-engine.yaml \
+            --var ref=${{ github.sha }}