Performance: use bulk request for docker stats (#7098)

src/utils/docker/stats-helpers.js

@@ -0,0 +1,33 @@
+export function calculateCPUPercent(stats) {
+  let cpuPercent = 0.0;
+  const cpuDelta = stats.cpu_stats.cpu_usage.total_usage - stats.precpu_stats.cpu_usage.total_usage;
+  const systemDelta = stats.cpu_stats.system_cpu_usage - stats.precpu_stats.system_cpu_usage;
+
+  if (systemDelta > 0.0 && cpuDelta > 0.0) {
+    cpuPercent = (cpuDelta / systemDelta) * stats.cpu_stats.online_cpus * 100.0;
+  }
+
+  return Math.round(cpuPercent * 10) / 10;
+}
+
+export function calculateUsedMemory(stats) {
+  // see https://github.com/docker/cli/blob/dcc161076861177b5eef6cb321722520db3184e7/cli/command/container/stats_helpers.go#L239
+  return (
+    stats.memory_stats.usage - (stats.memory_stats.total_inactive_file ?? stats.memory_stats.stats?.inactive_file ?? 0)
+  );
+}
+
+export function calculateThroughput(stats) {
+  let rxBytes = 0;
+  let txBytes = 0;
+  if (stats.networks?.network) {
+    rxBytes = stats.networks?.network.rx_bytes;
+    txBytes = stats.networks?.network.tx_bytes;
+  } else if (stats.networks && Array.isArray(Object.values(stats.networks))) {
+    Object.values(stats.networks).forEach((containerInterface) => {
+      rxBytes += containerInterface.rx_bytes;
+      txBytes += containerInterface.tx_bytes;
+    });
+  }
+  return { rxBytes, txBytes };
+}

src/utils/docker/stats-helpers.test.js

@@ -0,0 +1,55 @@
+import { describe, expect, it } from "vitest";
+
+import { calculateCPUPercent, calculateThroughput, calculateUsedMemory } from "./stats-helpers";
+
+describe("utils/docker/stats-helpers", () => {
+  it("calculateCPUPercent returns 0 when deltas are not positive", () => {
+    expect(
+      calculateCPUPercent({
+        cpu_stats: { cpu_usage: { total_usage: 100 }, system_cpu_usage: 1000, online_cpus: 2 },
+        precpu_stats: { cpu_usage: { total_usage: 100 }, system_cpu_usage: 1000 },
+      }),
+    ).toBe(0);
+  });
+
+  it("calculateCPUPercent computes percent and rounds to 1 decimal", () => {
+    // cpuDelta=100, systemDelta=1000, cpus=2 => (100/1000)*2*100 = 20.0
+    expect(
+      calculateCPUPercent({
+        cpu_stats: { cpu_usage: { total_usage: 200 }, system_cpu_usage: 2000, online_cpus: 2 },
+        precpu_stats: { cpu_usage: { total_usage: 100 }, system_cpu_usage: 1000 },
+      }),
+    ).toBe(20);
+  });
+
+  it("calculateUsedMemory subtracts inactive file (prefers total_inactive_file)", () => {
+    const stats = {
+      memory_stats: {
+        usage: 1000,
+        total_inactive_file: 100,
+        stats: { inactive_file: 200 },
+      },
+    };
+    expect(calculateUsedMemory(stats)).toBe(900);
+  });
+
+  it("calculateUsedMemory falls back to stats.inactive_file when total_inactive_file missing", () => {
+    const stats = {
+      memory_stats: {
+        usage: 1000,
+        stats: { inactive_file: 200 },
+      },
+    };
+    expect(calculateUsedMemory(stats)).toBe(800);
+  });
+
+  it("calculateThroughput uses the special networks.network key when present", () => {
+    const stats = { networks: { network: { rx_bytes: 5, tx_bytes: 6 }, eth0: { rx_bytes: 1, tx_bytes: 2 } } };
+    expect(calculateThroughput(stats)).toEqual({ rxBytes: 5, txBytes: 6 });
+  });
+
+  it("calculateThroughput sums all interfaces otherwise", () => {
+    const stats = { networks: { eth0: { rx_bytes: 1, tx_bytes: 2 }, eth1: { rx_bytes: 3, tx_bytes: 4 } } };
+    expect(calculateThroughput(stats)).toEqual({ rxBytes: 4, txBytes: 6 });
+  });
+});

src/utils/docker/stats.js

@@ -0,0 +1,92 @@
+import Docker from "dockerode";
+
+import { getSettings } from "utils/config/config";
+import getDockerArguments from "utils/config/docker";
+import { containersFromConfig, hasHomepageLabels } from "utils/config/service-helpers";
+import { calculateCPUPercent, calculateThroughput, calculateUsedMemory } from "utils/docker/stats-helpers";
+
+// mem and network are omitted when docker does not report them, so the widget can skip those blocks
+async function statsForContainer(docker, id) {
+  try {
+    const raw = await docker.getContainer(id).stats({ stream: false });
+    const stats = { cpu: calculateCPUPercent(raw) };
+
+    if (raw.memory_stats?.usage) {
+      stats.mem = calculateUsedMemory(raw);
+    }
+
+    if (raw.networks) {
+      const { rxBytes, txBytes } = calculateThroughput(raw);
+      stats.rx = rxBytes;
+      stats.tx = txBytes;
+    }
+
+    return stats;
+  } catch (e) {
+    // ensure one failed call does not look like an absent container
+    return { error: e?.message ?? "stats unavailable" };
+  }
+}
+
+export async function getDockerStats(server) {
+  const dockerArgs = getDockerArguments(server);
+  const docker = new Docker(dockerArgs.conn);
+
+  const [containers, configured] = await Promise.all([
+    docker.listContainers({ all: true }),
+    containersFromConfig(server),
+  ]);
+
+  if (!Array.isArray(containers)) {
+    return { error: "query failed" };
+  }
+
+  const { instanceName } = getSettings();
+  const targets = {};
+  const localIds = new Set();
+
+  containers.forEach((container) => {
+    localIds.add(container.Id);
+    if (container.State !== "running") return;
+
+    const labelled = hasHomepageLabels(container.Labels, instanceName);
+    container.Names.forEach((name) => {
+      const containerName = name.replace(/^\//, "");
+      if (labelled || configured.has(containerName)) targets[containerName] = container.Id;
+    });
+  });
+
+  if (dockerArgs.swarm) {
+    const [services, tasks] = await Promise.all([
+      docker.listServices().catch(() => []),
+      docker.listTasks({ filters: { "desired-state": ["running"] } }).catch(() => []),
+    ]);
+
+    const tasksByService = {};
+    tasks.forEach((task) => {
+      (tasksByService[task.ServiceID] ??= []).push(task);
+    });
+
+    services.forEach((service) => {
+      const name = service.Spec?.Name;
+      if (!name || targets[name]) return;
+      if (!configured.has(name) && !hasHomepageLabels(service.Spec?.Labels, instanceName)) return;
+
+      // stats are only available for containers running on this node
+      const serviceTasks = tasksByService[service.ID] ?? [];
+      const task = serviceTasks.find((candidate) => localIds.has(candidate.Status?.ContainerStatus?.ContainerID));
+      const containerId = task?.Status?.ContainerStatus?.ContainerID;
+      if (containerId) targets[name] = containerId;
+    });
+  }
+
+  const names = Object.keys(targets);
+  const results = await Promise.all(names.map((name) => statsForContainer(docker, targets[name])));
+
+  const stats = {};
+  names.forEach((name, index) => {
+    stats[name] = results[index];
+  });
+
+  return { stats };
+}

src/utils/docker/stats.test.js

@@ -0,0 +1,173 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { state, DockerCtor, getDockerArguments, containersFromConfig, hasHomepageLabels, getSettings } = vi.hoisted(
+  () => {
+    const state = {
+      docker: null,
+      containers: [],
+      statsById: {},
+      dockerArgs: { conn: { socketPath: "/var/run/docker.sock" }, swarm: false },
+    };
+
+    function DockerCtor() {
+      return state.docker;
+    }
+
+    return {
+      state,
+      DockerCtor,
+      getDockerArguments: vi.fn(() => state.dockerArgs),
+      containersFromConfig: vi.fn(async () => new Set()),
+      hasHomepageLabels: vi.fn(() => false),
+      getSettings: vi.fn(() => ({ instanceName: undefined })),
+    };
+  },
+);
+
+vi.mock("dockerode", () => ({ default: DockerCtor }));
+vi.mock("utils/config/docker", () => ({ default: getDockerArguments }));
+vi.mock("utils/config/service-helpers", () => ({ containersFromConfig, hasHomepageLabels }));
+vi.mock("utils/config/config", () => ({ getSettings }));
+
+import { getDockerStats } from "./stats";
+
+// cpu 20%, mem 900, rx 4, tx 6
+const rawStats = (overrides = {}) => ({
+  cpu_stats: { cpu_usage: { total_usage: 200 }, system_cpu_usage: 2000, online_cpus: 2 },
+  precpu_stats: { cpu_usage: { total_usage: 100 }, system_cpu_usage: 1000 },
+  memory_stats: { usage: 1000, total_inactive_file: 100 },
+  networks: { eth0: { rx_bytes: 1, tx_bytes: 2 }, eth1: { rx_bytes: 3, tx_bytes: 4 } },
+  ...overrides,
+});
+
+describe("utils/docker/stats", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    state.dockerArgs = { conn: { socketPath: "/var/run/docker.sock" }, swarm: false };
+    state.containers = [];
+    state.statsById = {};
+    state.docker = {
+      listContainers: vi.fn(async () => state.containers),
+      listServices: vi.fn(async () => []),
+      listTasks: vi.fn(async () => []),
+      getContainer: vi.fn((id) => ({
+        stats: vi.fn(async () => {
+          const entry = state.statsById[id];
+          if (entry instanceof Error) throw entry;
+          if (!entry) throw new Error(`no stats for ${id}`);
+          return entry;
+        }),
+      })),
+    };
+    containersFromConfig.mockResolvedValue(new Set());
+    hasHomepageLabels.mockReturnValue(false);
+    getSettings.mockReturnValue({ instanceName: undefined });
+  });
+
+  it("returns computed stats for configured running containers", async () => {
+    containersFromConfig.mockResolvedValue(new Set(["app"]));
+    state.containers = [{ Names: ["/app"], Id: "cid1", State: "running" }];
+    state.statsById.cid1 = rawStats();
+
+    expect(await getDockerStats("local")).toEqual({ stats: { app: { cpu: 20, mem: 900, rx: 4, tx: 6 } } });
+    expect(getDockerArguments).toHaveBeenCalledWith("local");
+    expect(containersFromConfig).toHaveBeenCalledWith("local");
+  });
+
+  it("omits containers that are neither configured nor labelled", async () => {
+    containersFromConfig.mockResolvedValue(new Set(["app"]));
+    state.containers = [
+      { Names: ["/app"], Id: "cid1", State: "running" },
+      { Names: ["/secret-db"], Id: "cid2", State: "running" },
+    ];
+    state.statsById.cid1 = rawStats();
+    state.statsById.cid2 = rawStats();
+
+    const result = await getDockerStats("local");
+
+    expect(Object.keys(result.stats)).toEqual(["app"]);
+    expect(state.docker.getContainer).not.toHaveBeenCalledWith("cid2");
+  });
+
+  it("includes containers discovered through homepage labels", async () => {
+    hasHomepageLabels.mockImplementation((labels) => labels?.["homepage.name"] !== undefined);
+    state.containers = [{ Names: ["/labelled"], Id: "cid1", State: "running", Labels: { "homepage.name": "App" } }];
+    state.statsById.cid1 = rawStats();
+
+    expect(Object.keys((await getDockerStats("local")).stats)).toEqual(["labelled"]);
+  });
+
+  it("does not collect stats for containers that are not running", async () => {
+    containersFromConfig.mockResolvedValue(new Set(["app"]));
+    state.containers = [{ Names: ["/app"], Id: "cid1", State: "exited" }];
+
+    expect(await getDockerStats("local")).toEqual({ stats: {} });
+    expect(state.docker.getContainer).not.toHaveBeenCalled();
+  });
+
+  it("keeps a per container stats failure without failing the rest", async () => {
+    containersFromConfig.mockResolvedValue(new Set(["ok", "broken"]));
+    state.containers = [
+      { Names: ["/ok"], Id: "cid1", State: "running" },
+      { Names: ["/broken"], Id: "cid2", State: "running" },
+    ];
+    state.statsById.cid1 = rawStats();
+    state.statsById.cid2 = new Error("stats unavailable");
+
+    expect(await getDockerStats("local")).toEqual({
+      stats: { ok: { cpu: 20, mem: 900, rx: 4, tx: 6 }, broken: { error: "stats unavailable" } },
+    });
+  });
+
+  it("omits mem and network when docker does not report them", async () => {
+    containersFromConfig.mockResolvedValue(new Set(["app"]));
+    state.containers = [{ Names: ["/app"], Id: "cid1", State: "running" }];
+    state.statsById.cid1 = rawStats({ memory_stats: {}, networks: undefined });
+
+    expect(await getDockerStats("local")).toEqual({ stats: { app: { cpu: 20 } } });
+  });
+
+  it("returns an error when docker returns a non-array containers payload", async () => {
+    state.containers = Buffer.from("bad");
+
+    expect(await getDockerStats("local")).toEqual({ error: "query failed" });
+  });
+
+  it("resolves a swarm service through its local task container", async () => {
+    state.dockerArgs.swarm = true;
+    containersFromConfig.mockResolvedValue(new Set(["svc"]));
+    state.containers = [{ Names: ["/other"], Id: "local1", State: "running" }];
+    state.statsById.local1 = rawStats();
+    state.docker.listServices.mockResolvedValue([{ ID: "sid", Spec: { Name: "svc" } }]);
+    state.docker.listTasks.mockResolvedValue([
+      { ServiceID: "sid", Status: { ContainerStatus: { ContainerID: "remote1" } } },
+      { ServiceID: "sid", Status: { ContainerStatus: { ContainerID: "local1" } } },
+    ]);
+
+    expect(await getDockerStats("swarm")).toEqual({ stats: { svc: { cpu: 20, mem: 900, rx: 4, tx: 6 } } });
+  });
+
+  it("skips a swarm service with no container on this node", async () => {
+    state.dockerArgs.swarm = true;
+    containersFromConfig.mockResolvedValue(new Set(["svc"]));
+    state.containers = [{ Names: ["/other"], Id: "local1", State: "running" }];
+    state.docker.listServices.mockResolvedValue([{ ID: "sid", Spec: { Name: "svc" } }]);
+    state.docker.listTasks.mockResolvedValue([
+      { ServiceID: "sid", Status: { ContainerStatus: { ContainerID: "remote1" } } },
+    ]);
+
+    expect(await getDockerStats("swarm")).toEqual({ stats: {} });
+  });
+
+  it("omits swarm services that are neither configured nor labelled", async () => {
+    state.dockerArgs.swarm = true;
+    state.containers = [{ Names: ["/other"], Id: "local1", State: "running" }];
+    state.statsById.local1 = rawStats();
+    state.docker.listServices.mockResolvedValue([{ ID: "sid", Spec: { Name: "internal" } }]);
+    state.docker.listTasks.mockResolvedValue([
+      { ServiceID: "sid", Status: { ContainerStatus: { ContainerID: "local1" } } },
+    ]);
+
+    expect(await getDockerStats("swarm")).toEqual({ stats: {} });
+  });
+});