esh-pfi-infrastructure/playbooks/esh-matter-lxc.yaml

# esh-matter — a small LXC on esh-pve whose ONLY network leg is VLAN 90 (esh-iot),
# running the Matter server (matter.js, stacks/matter-server) for Home Assistant.
# Requested by ha-dev, operator-approved 2026-09-26.
#
# WHY IT SITS ON VLAN 90: Matter's operational traffic is IPv6, and VLAN 90's only
# IPv6 is the Thread ULA fdad:29e:d492:fd87::/64. It is advertised by an Echo
# border router, not the UDM, and is visible only on that link. The Thread routes
# (RA route-information options from the border routers) are likewise link-only.
# HA core reaches the server over an IPv4 websocket (routed VLAN 50 → 90; the UDM
# policy InternalToIOT already allows it), so HA itself does not change.
#
# WHY AN LXC, NOT A MACVLAN ON esh-docker-vm: it keeps the untrusted IoT leg off the
# host that runs HA and everything else (a VM with a history of wedges). It gets
# its own firewall and its own vzdump backup.
#
# IPv6: kernel RA processing (no NetworkManager/networkd; the Debian 12 template
# uses ifupdown). accept_ra=1 plus accept_ra_rt_info_max_plen=64 learns Thread
# routes. IPv6 forwarding stays OFF (matter.js os_requirements: forwarding
# disables RFC 4191 reachability probing). Docker is told not to touch forwarding
# or iptables; the server runs with host networking, so it needs neither.
#
# FIREWALL (in-CT nftables): 5580 (the websocket and dashboard, UNAUTHENTICATED) is
# accepted only from HA at {{ ha_ip }}, then dropped for everyone else, v4 and v6.
# SSH is accepted only from non-IoT management ranges. Everything else, Matter
# UDP 5540 and mDNS 5353 included, is left alone on purpose. That keeps conntrack
# out of the Matter path, so sleepy-device reports are not dropped by a 120 s UDP
# timeout (matter.js "Stateful firewalls" note).
#
# BACKUP: the CT is added to esh-pve's vzdump job (PBS-ANA, mirrored to PBS-NH3).
# The data dir holds the Matter fabric root credentials; losing it means
# re-commissioning every device.
#
# Run: scripts/elway root@esh-pve --playbook playbooks/esh-matter-lxc.yaml
# Then: scripts/deploy-stack.sh esh-matter matter-server  (+ docker compose up -d)

vars:
  ctid: 111
  hostname: esh-matter
  ip_cidr: 10.0.90.20/24
  ip_addr: 10.0.90.20
  gateway: 10.0.90.1
  vlan: 90
  rootfs_storage: local-lvm
  rootfs_gb: 8
  cores: 2
  memory_mb: 1024
  swap_mb: 512
  startup_order: 30
  template: debian-12-standard_12.12-1_amd64.tar.zst
  ha_ip: 10.0.50.46
  # Non-IoT sources allowed to SSH: esh-server, esh-userland, esh-mgmt, NH3, mesh.
  ssh_sources: "10.0.50.0/24, 10.0.10.0/24, 10.0.250.0/24, 10.100.0.0/16, 100.64.0.0/10"
  infra_ops_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet"

steps:
  - name: Create CT {{ ctid }} ({{ hostname }}) with its only leg on VLAN {{ vlan }}, IPv6 SLAAC
    shell: |
      pct create {{ ctid }} local:vztmpl/{{ template }} \
        --hostname {{ hostname }} --unprivileged 1 --features nesting=1,keyctl=1 \
        --cores {{ cores }} --memory {{ memory_mb }} --swap {{ swap_mb }} \
        --rootfs {{ rootfs_storage }}:{{ rootfs_gb }} \
        --net0 name=eth0,bridge=vmbr0,firewall=0,gw={{ gateway }},ip={{ ip_cidr }},ip6=auto,tag={{ vlan }},type=veth \
        --nameserver {{ gateway }} \
        --onboot 1 --startup order={{ startup_order }},up=10 \
        --description "{{ hostname }} — Matter server (matter.js) on VLAN {{ vlan }} for Home Assistant. Built by eshpfi playbooks/esh-matter-lxc.yaml; stack stacks/matter-server. IN vzdump (fabric credentials)."
    when: "! pct status {{ ctid }} >/dev/null 2>&1"

  # esh-pve's job names its vmids explicitly. Append ours; leave the rest alone.
  - name: Add CT {{ ctid }} to esh-pve's vzdump job
    shell: |
      set -e
      CT={{ ctid }} JOBS="$(pvesh get /cluster/backup --output-format json)" python3 - <<'PY'
      import json, os, subprocess
      ct = os.environ["CT"]
      for j in json.loads(os.environ["JOBS"]):
          ids = [x for x in str(j.get("vmid", "")).split(",") if x]
          if not ids or ct in ids:
              continue
          subprocess.run(["pvesh", "set", "/cluster/backup/" + j["id"], "--vmid", ",".join(ids + [ct])], check=True)
          print("added", ct, "to", j["id"])
      PY
    when: "! grep -E '^\\s+vmid .*\\b{{ ctid }}\\b' /etc/pve/jobs.cfg"

  - name: Start the container
    shell: pct start {{ ctid }} && sleep 6
    when: "! pct status {{ ctid }} | grep -q running"

  - name: IPv6 RA processing with Thread route-information options, forwarding off
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      cat > /etc/sysctl.d/60-matter-ipv6.conf <<'EOC'
      # Matter server — see eshpfi playbooks/esh-matter-lxc.yaml
      net.ipv6.conf.all.forwarding = 0
      net.ipv6.conf.eth0.accept_ra = 1
      net.ipv6.conf.eth0.accept_ra_rt_info_max_plen = 64
      EOC
      sysctl -q -p /etc/sysctl.d/60-matter-ipv6.conf
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'test $(cat /proc/sys/net/ipv6/conf/eth0/accept_ra_rt_info_max_plen) = 64'"

  - name: Base packages + bookworm point upgrade (+ nftables)
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      for i in $(seq 1 30); do getent hosts deb.debian.org >/dev/null && break; sleep 1; done
      apt-get update -qq
      apt-get -y -qq full-upgrade
      apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg sudo jq less rsync locales nftables iputils-ping iproute2
      sed -i 's/^# *en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen >/dev/null
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'command -v nft && command -v rsync && command -v jq && locale -a | grep -qi en_US.utf8' >/dev/null 2>&1"

  - name: Fleet identities (docker 851, infra-ops 850, vh 1000) + /opt/docker tree
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      getent group docker >/dev/null || groupadd -g 851 docker
      getent group infra-ops >/dev/null || groupadd -g 850 infra-ops
      id infra-ops >/dev/null 2>&1 || useradd -u 850 -g 850 -G docker -m -s /bin/bash infra-ops
      chmod 0700 /home/infra-ops
      install -d -m 0700 -o infra-ops -g infra-ops /home/infra-ops/.ssh
      echo '{{ infra_ops_pubkey }}' > /home/infra-ops/.ssh/authorized_keys
      chown infra-ops:infra-ops /home/infra-ops/.ssh/authorized_keys
      chmod 0600 /home/infra-ops/.ssh/authorized_keys
      echo 'infra-ops ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/infra-ops
      chmod 0440 /etc/sudoers.d/infra-ops
      id vh >/dev/null 2>&1 || useradd -u 1000 -U -G docker,sudo -m -s /bin/bash vh
      chmod 0700 /home/vh
      install -d -m 2775 -o root -g docker /opt/docker /opt/docker/compose /opt/docker/conf
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'test \"$(id -u infra-ops)\" = 850 && test \"$(getent group docker | cut -d: -f3)\" = 851 && test -d /opt/docker/compose'"

  # Applied BEFORE Docker, so it is already in force when dockerd first starts.
  - name: Firewall — 5580 from HA only, SSH from management ranges only
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      cat > /etc/nftables.conf <<'EOC'
      #!/usr/sbin/nft -f
      # esh-matter guard — eshpfi playbooks/esh-matter-lxc.yaml. Policy ACCEPT on
      # purpose: only the two ports below are filtered, so Matter UDP and mDNS never
      # touch conntrack-based rules.
      flush ruleset
      table inet matter_guard {
        chain input {
          type filter hook input priority 0; policy accept;
          iif "lo" accept
          tcp dport 5580 ip saddr {{ ha_ip }} accept
          tcp dport 5580 counter drop
          tcp dport 22 ip saddr { {{ ssh_sources }} } accept
          tcp dport 22 counter drop
        }
      }
      EOC
      systemctl enable -q nftables
      systemctl restart nftables
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'nft list table inet matter_guard 2>/dev/null | grep -q \"5580 ip saddr {{ ha_ip }} accept\"'"

  - name: docker-ce, hands off forwarding and iptables (host networking only)
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      install -d /etc/docker
      printf '{\n  "ip-forward": false,\n  "iptables": false,\n  "ip6tables": false\n}\n' > /etc/docker/daemon.json
      install -m 0755 -d /etc/apt/keyrings
      curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
      chmod a+r /etc/apt/keyrings/docker.asc
      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" \
        > /etc/apt/sources.list.d/docker.list
      apt-get update -qq
      apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-compose-plugin
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'command -v docker' >/dev/null 2>&1"

  - name: Matter data dir (fabric credentials) owned by the container's uid 1000
    shell: pct exec {{ ctid }} -- install -d -m 0750 -o 1000 -g 1000 /opt/docker/data /opt/docker/data/matter-server
    when: "! pct exec {{ ctid }} -- test -d /opt/docker/data/matter-server"

verify:
  - name: Running, onboot, in the vzdump job
    shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && grep -Eq '^\\s+vmid .*\\b{{ ctid }}\\b' /etc/pve/jobs.cfg"
    changed_when: "false"

  - name: IPv6 — RA route-info on, forwarding off, a SLAAC address on eth0
    shell: |
      pct exec {{ ctid }} -- sh -c 'test $(cat /proc/sys/net/ipv6/conf/eth0/accept_ra_rt_info_max_plen) = 64 && test $(cat /proc/sys/net/ipv6/conf/all/forwarding) = 0 && ip -6 addr show dev eth0 scope global | grep -q inet6'
    changed_when: "false"

  - name: Firewall loaded and persistent
    shell: pct exec {{ ctid }} -- sh -c 'nft list table inet matter_guard | grep -Eq "dport 5580 counter .*drop" && systemctl is-enabled --quiet nftables'
    changed_when: "false"

  - name: Docker left forwarding alone
    shell: pct exec {{ ctid }} -- sh -c 'docker info >/dev/null && test $(cat /proc/sys/net/ipv6/conf/all/forwarding) = 0'
    changed_when: "false"

  - name: Fleet identities are the pinned ids
    shell: |
      pct exec {{ ctid }} -- sh -c 'test "$(id -u infra-ops)" = 850 && test "$(id -u vh)" = 1000 && test "$(getent group docker | cut -d: -f3)" = 851'
    changed_when: "false"