Merge pull request #1184 from adamoutler/devving-devcontainer

feat: Devcontainer
2025-12-07 01:26:11 -08:00 · 2025-09-21 10:08:51 +10:00
parent 599bedf908 2d1a9da046
commit 83aa1a961e
19 changed files with 815 additions and 0 deletions
--- a/.coverage
+++ b/.coverage
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -0,0 +1,112 @@
+# DO NOT MODIFY THIS FILE DIRECTLY. IT IS AUTO-GENERATED BY .devcontainer/scripts/generate-dockerfile.sh
+
+# ---/Dockerfile---
+FROM alpine:3.22 AS builder
+
+ARG INSTALL_DIR=/app
+
+ENV PYTHONUNBUFFERED=1
+
+# Install build dependencies
+RUN apk add --no-cache bash shadow python3 python3-dev gcc musl-dev libffi-dev openssl-dev git \
+    && python -m venv /opt/venv
+
+# Enable venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+
+RUN pip install openwrt-luci-rpc asusrouter asyncio aiohttp graphene flask flask-cors unifi-sm-api tplink-omada-client wakeonlan pycryptodome requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet python-nmap dnspython librouteros yattag git+https://github.com/foreign-sub/aiofreepybox.git
+
+# Append Iliadbox certificate to aiofreepybox
+
+# second stage
+FROM alpine:3.22 AS runner
+
+ARG INSTALL_DIR=/app
+
+COPY --from=builder /opt/venv /opt/venv
+COPY --from=builder /usr/sbin/usermod /usr/sbin/groupmod /usr/sbin/
+
+# Enable venv
+ENV PATH="/opt/venv/bin:$PATH" 
+
+# default port and listen address
+ENV PORT=20211 LISTEN_ADDR=0.0.0.0 
+
+# needed for s6-overlay
+ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0
+
+# ❗ IMPORTANT - if you modify this file modify the /install/install_dependecies.sh file as well ❗ 
+
+RUN apk update --no-cache \
+    && apk add --no-cache bash libbsd zip lsblk gettext-envsubst sudo mtr tzdata s6-overlay \
+    && apk add --no-cache curl arp-scan iproute2 iproute2-ss nmap nmap-scripts traceroute nbtscan avahi avahi-tools openrc dbus net-tools net-snmp-tools bind-tools awake ca-certificates \
+    && apk add --no-cache sqlite php83 php83-fpm php83-cgi php83-curl php83-sqlite3 php83-session \
+    && apk add --no-cache python3 nginx \
+    && ln -s /usr/bin/awake /usr/bin/wakeonlan \
+    && rm -f /etc/nginx/http.d/default.conf
+
+
+# Add crontab file
+COPY --chmod=600 --chown=root:root install/crontab /etc/crontabs/root
+
+# Start all required services
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=2 \
+    CMD curl -sf -o /dev/null ${LISTEN_ADDR}:${PORT}/php/server/query_json.php?file=app_state.json
+
+ENTRYPOINT ["/init"]
+
+# ---/resources/devcontainer-Dockerfile---
+
+# Devcontainer build stage (do not build directly)
+# This file is combined with the root /Dockerfile by
+#   .devcontainer/scripts/generate-dockerfile.sh
+# The generator appends this stage to produce .devcontainer/Dockerfile.
+# Prefer to place dev-only setup here; use setup.sh only for runtime fixes.
+
+FROM runner AS devcontainer
+ENV INSTALL_DIR=/app 
+ENV PYTHONPATH=/workspaces/NetAlertX/test:/workspaces/NetAlertX/server:/app:/app/server:/opt/venv/lib/python3.12/site-packages
+
+# Install common tools, create user, and set up sudo
+RUN apk add --no-cache git nano vim jq php83-pecl-xdebug py3-pip nodejs sudo gpgconf pytest pytest-cov && \
+    adduser -D -s /bin/sh netalertx && \
+    addgroup netalertx nginx && \
+    addgroup netalertx www-data && \
+    echo "netalertx ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-netalertx && \
+    chmod 440 /etc/sudoers.d/90-netalertx
+# Install debugpy in the virtualenv if present, otherwise into system python3
+RUN /bin/sh -c '(/opt/venv/bin/python3 -m pip install --no-cache-dir debugpy) || (python3 -m pip install --no-cache-dir debugpy) || true'
+# setup nginx
+COPY .devcontainer/resources/netalertx-devcontainer.conf /etc/nginx/http.d/netalert-frontend.conf
+RUN set -e; \
+    chown netalertx:nginx /etc/nginx/http.d/netalert-frontend.conf; \
+    install -d -o netalertx -g www-data -m 775 /app; \
+    install -d -o netalertx -g www-data -m 755 /run/nginx; \
+    install -d -o netalertx -g www-data -m 755 /var/lib/nginx/logs; \
+    rm -f /var/lib/nginx/logs/* || true; \
+    for f in error access; do : > /var/lib/nginx/logs/$f.log; done; \
+    install -d -o netalertx -g www-data -m 777 /run/php; \
+    install -d -o netalertx -g www-data -m 775 /var/log/php; \
+    chown -R netalertx:www-data /etc/nginx/http.d; \
+    chmod -R 775 /etc/nginx/http.d; \
+    chown -R netalertx:www-data /var/lib/nginx; \
+    chmod -R 755 /var/lib/nginx && \
+    chown -R netalertx:www-data /var/log/nginx/ && \
+    sed -i '/^user /d' /etc/nginx/nginx.conf; \
+    sed -i 's|^error_log .*|error_log /dev/stderr warn;|' /etc/nginx/nginx.conf; \
+    sed -i 's|^access_log .*|access_log /dev/stdout main;|' /etc/nginx/nginx.conf; \
+    sed -i 's|error_log .*|error_log /dev/stderr warn;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \
+    sed -i 's|access_log .*|access_log /dev/stdout main;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \
+    mkdir -p /run/openrc; \
+    chown netalertx:nginx /run/openrc/; \
+    rm -Rf /run/openrc/*;
+
+# setup pytest
+RUN sudo /opt/venv/bin/python -m pip install -U pytest pytest-cov
+
+WORKDIR /workspaces/NetAlertX
+
+
+ENTRYPOINT ["/bin/sh","-c","sleep infinity"]
--- a/.devcontainer/README.md
+++ b/.devcontainer/README.md
@@ -0,0 +1,30 @@
+# NetAlertX Devcontainer Notes
+
+This devcontainer replicates the production container as closely as practical, with a few development-oriented differences.
+
+Key behavior
+- No init process: Services are managed by shell scripts using killall, setsid, and nohup. Startup and restarts are script-driven rather than supervised by an init system.
+- Autogenerated Dockerfile: The effective devcontainer Dockerfile is generated on demand by `.devcontainer/scripts/generate-dockerfile.sh`. It combines the root `Dockerfile` (with certain COPY instructions removed) and an extra "devcontainer" stage from `.devcontainer/resources/devcontainer-Dockerfile`. When you change the resource Dockerfile, re-run the generator to refresh `.devcontainer/Dockerfile`.
+- Where to put setup: Prefer baking setup into `.devcontainer/resources/devcontainer-Dockerfile`. Use `.devcontainer/scripts/setup.sh` only for steps that must happen at container start (e.g., cleaning up nginx/php ownership, creating directories, touching runtime files) or depend on runtime paths.
+
+Debugging (F5)
+The Frontend and backend run in debug mode always.  You can attach your debugger at any time.  
+- Python Backend Debug: Attach - The backend runs with a debugger on port 5678. Set breakpoints in the code and press F5 to begin triggering them.
+- PHP Frontend (XDebug) Xdebug listens on 9003. Start listening and use an Xdebug extension in your browser to debug PHP.
+
+Common workflows (F1->Tasks: Run Task)
+- Regenerate the devcontainer Dockerfile: Run the VS Code task "Generate Dockerfile" or execute `.devcontainer/scripts/generate-dockerfile.sh`. The result is `.devcontainer/Dockerfile`.
+- Re-run startup provisioning: Use the task "Re-Run Startup Script" to execute `.devcontainer/scripts/setup.sh` in the container.
+- Start services:
+  - Backend (GraphQL/Flask): `.devcontainer/scripts/restart-backend.sh` starts it under debugpy and logs to `/app/log/app.log`
+  - Frontend (nginx + PHP-FPM): Started via setup.sh; can be restarted by the task "Start Frontend (nginx and PHP-FPM)".
+
+Testing
+- pytest is installed via Alpine packages (py3-pytest, py3-pytest-cov).
+- PYTHONPATH includes workspace and venv site-packages so tests can import `server/*` modules and third-party libs.
+- Run tests via VS Code Pytest Runner or `pytest -q` from the workspace root.
+
+Conventions
+- Don’t edit `.devcontainer/Dockerfile` directly; edit `.devcontainer/resources/devcontainer-Dockerfile` and regenerate.
+- Keep setup in the resource Dockerfile when possible; reserve `setup.sh` for runtime fixes.
+- Avoid hardcoding ports/secrets; prefer existing settings and helpers (see `.github/copilot-instructions.md`).
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,75 @@
+{
+  "name": "NetAlertX DevContainer",
+  "remoteUser": "netalertx",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": "..",
+    "target": "devcontainer"
+  },
+  "workspaceFolder": "/workspaces/NetAlertX",
+
+  "runArgs": [
+    "--privileged",
+    "--cap-add=NET_ADMIN",
+    "--cap-add=NET_RAW",
+    // Ensure containers can resolve host.docker.internal to the host (required on Linux)
+    "--add-host=host.docker.internal:host-gateway"
+  ],
+  
+  "postStartCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/setup.sh",
+  
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-azuretools.vscode-docker",
+        "felixfbecker.php-debug",
+        "bmewburn.vscode-intelephense-client",
+        "xdebug.php-debug",
+        "ms-python.vscode-pylance",
+        "pamaron.pytest-runner",
+        "coderabbit.coderabbit-vscode",
+        "ms-python.black-formatter"
+      ]
+      ,
+      "settings": {
+        "terminal.integrated.cwd": "${containerWorkspaceFolder}",
+        // Python testing configuration
+        "python.testing.pytestEnabled": true,
+        "python.testing.unittestEnabled": false,
+        "python.testing.pytestArgs": [
+          "test"
+        ],
+        // Make sure we discover tests and import server correctly
+        "python.analysis.extraPaths": [
+          "/workspaces/NetAlertX",
+          "/workspaces/NetAlertX/server",
+          "/app",
+          "/app/server"
+        ]
+      }
+    }
+  },
+  "forwardPorts": [5678, 9000, 9003, 20211, 20212],
+
+  "portsAttributes": {
+    "20211": {
+      "label": "Frontend:Nginx+PHP"
+    },
+    "20212": {
+      "label": "Backend:GraphQL"
+    },
+    "9003": {
+      "label": "PHP Debug:Xdebug"
+    },
+    "9000": {
+      "label": "PHP-FPM:FastCGI"
+    },
+    "5678": {
+      "label": "Python Debug:debugpy"
+    }
+  },
+
+  // Optional: ensures compose services are stopped when you close the window
+  "shutdownAction": "stopContainer"
+}
--- a/.devcontainer/resources/99-xdebug.ini
+++ b/.devcontainer/resources/99-xdebug.ini
@@ -0,0 +1,8 @@
+zend_extension="xdebug.so"
+[xdebug]
+xdebug.mode=develop,debug
+xdebug.log_level=0
+xdebug.client_host=host.docker.internal
+xdebug.client_port=9003
+xdebug.start_with_request=yes
+xdebug.discover_client_host=1
--- a/.devcontainer/resources/devcontainer-Dockerfile
+++ b/.devcontainer/resources/devcontainer-Dockerfile
@@ -0,0 +1,51 @@
+# Devcontainer build stage (do not build directly)
+# This file is combined with the root /Dockerfile by
+#   .devcontainer/scripts/generate-dockerfile.sh
+# The generator appends this stage to produce .devcontainer/Dockerfile.
+# Prefer to place dev-only setup here; use setup.sh only for runtime fixes.
+
+FROM runner AS devcontainer
+ENV INSTALL_DIR=/app 
+ENV PYTHONPATH=/workspaces/NetAlertX/test:/workspaces/NetAlertX/server:/app:/app/server:/opt/venv/lib/python3.12/site-packages
+
+# Install common tools, create user, and set up sudo
+RUN apk add --no-cache git nano vim jq php83-pecl-xdebug py3-pip nodejs sudo gpgconf pytest pytest-cov && \
+    adduser -D -s /bin/sh netalertx && \
+    addgroup netalertx nginx && \
+    addgroup netalertx www-data && \
+    echo "netalertx ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-netalertx && \
+    chmod 440 /etc/sudoers.d/90-netalertx
+# Install debugpy in the virtualenv if present, otherwise into system python3
+RUN /bin/sh -c '(/opt/venv/bin/python3 -m pip install --no-cache-dir debugpy) || (python3 -m pip install --no-cache-dir debugpy) || true'
+# setup nginx
+COPY .devcontainer/resources/netalertx-devcontainer.conf /etc/nginx/http.d/netalert-frontend.conf
+RUN set -e; \
+    chown netalertx:nginx /etc/nginx/http.d/netalert-frontend.conf; \
+    install -d -o netalertx -g www-data -m 775 /app; \
+    install -d -o netalertx -g www-data -m 755 /run/nginx; \
+    install -d -o netalertx -g www-data -m 755 /var/lib/nginx/logs; \
+    rm -f /var/lib/nginx/logs/* || true; \
+    for f in error access; do : > /var/lib/nginx/logs/$f.log; done; \
+    install -d -o netalertx -g www-data -m 777 /run/php; \
+    install -d -o netalertx -g www-data -m 775 /var/log/php; \
+    chown -R netalertx:www-data /etc/nginx/http.d; \
+    chmod -R 775 /etc/nginx/http.d; \
+    chown -R netalertx:www-data /var/lib/nginx; \
+    chmod -R 755 /var/lib/nginx && \
+    chown -R netalertx:www-data /var/log/nginx/ && \
+    sed -i '/^user /d' /etc/nginx/nginx.conf; \
+    sed -i 's|^error_log .*|error_log /dev/stderr warn;|' /etc/nginx/nginx.conf; \
+    sed -i 's|^access_log .*|access_log /dev/stdout main;|' /etc/nginx/nginx.conf; \
+    sed -i 's|error_log .*|error_log /dev/stderr warn;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \
+    sed -i 's|access_log .*|access_log /dev/stdout main;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \
+    mkdir -p /run/openrc; \
+    chown netalertx:nginx /run/openrc/; \
+    rm -Rf /run/openrc/*;
+
+# setup pytest
+RUN sudo /opt/venv/bin/python -m pip install -U pytest pytest-cov
+
+WORKDIR /workspaces/NetAlertX
+
+
+ENTRYPOINT ["/bin/sh","-c","sleep infinity"]
--- a/.devcontainer/resources/netalertx-devcontainer.conf
+++ b/.devcontainer/resources/netalertx-devcontainer.conf
@@ -0,0 +1,26 @@
+log_format netalertx '$remote_addr - $remote_user [$time_local] "$request" '
+                     '$status $body_bytes_sent "$http_referer" '
+                     '"$http_user_agent" "$http_x_forwarded_for"';
+access_log /var/log/nginx/access.log netalertx flush=1s;
+error_log  /var/log/nginx/error.log  warn;
+
+server {
+    listen 20211 default_server;
+    root /app/front;
+    index index.php;
+
+    add_header X-Forwarded-Prefix "/netalertx" always;
+    proxy_set_header X-Forwarded-Prefix "/netalertx";
+
+    location ~* \.php$ {
+        add_header Cache-Control "no-store";
+        fastcgi_pass 127.0.0.1:9000;
+        include         fastcgi_params;
+        fastcgi_param   SCRIPT_FILENAME    $document_root$fastcgi_script_name;
+        fastcgi_param   SCRIPT_NAME        $fastcgi_script_name;
+        fastcgi_param   PHP_VALUE "xdebug.remote_enable=1";
+        fastcgi_connect_timeout 75;
+        fastcgi_send_timeout 600;
+        fastcgi_read_timeout 600;
+    }
+}
--- a/.devcontainer/scripts/generate-dockerfile.sh
+++ b/.devcontainer/scripts/generate-dockerfile.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+
+# Generator for .devcontainer/Dockerfile
+# Combines the root /Dockerfile (with some COPY lines removed) and
+# the dev-only stage in .devcontainer/resources/devcontainer-Dockerfile.
+# Run this script after modifying the resource Dockerfile to refresh
+# the final .devcontainer/Dockerfile used by the devcontainer.
+
+# Make a copy of the original Dockerfile to the .devcontainer folder
+# but remove the COPY . ${INSTALL_DIR}/ command from it. This avoids
+# overwriting /app (which uses symlinks to the workspace) and preserves
+# debugging capabilities inside the devcontainer.
+
+SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"
+DEVCONTAINER_DIR="${SCRIPT_DIR%/scripts}"
+ROOT_DIR="${DEVCONTAINER_DIR%/.devcontainer}"
+
+OUT_FILE="${DEVCONTAINER_DIR}/Dockerfile"
+
+echo "# DO NOT MODIFY THIS FILE DIRECTLY. IT IS AUTO-GENERATED BY .devcontainer/scripts/generate-dockerfile.sh" > "$OUT_FILE"
+echo "" >> "$OUT_FILE"
+echo "# ---/Dockerfile---" >> "$OUT_FILE"
+
+sed '/${INSTALL_DIR}/d' "${ROOT_DIR}/Dockerfile" >> "$OUT_FILE"
+
+# sed the line https://github.com/foreign-sub/aiofreepybox.git \\ to remove trailing backslash
+sed -i '/aiofreepybox.git/ s/ \\$//' "$OUT_FILE"
+
+# don't cat the file, just copy it in because it doesn't exist at build time
+sed -i 's|^ RUN cat ${INSTALL_DIR}/install/freebox_certificate.pem >> /opt/venv/lib/python3.12/site-packages/aiofreepybox/freebox_certificates.pem$| COPY install/freebox_certificate.pem /opt/venv/lib/python3.12/site-packages/aiofreepybox/freebox_certificates.pem |' "$OUT_FILE"
+
+echo "" >> "$OUT_FILE"
+echo "# ---/resources/devcontainer-Dockerfile---" >> "$OUT_FILE"
+echo "" >> "$OUT_FILE"
+
+cat "${DEVCONTAINER_DIR}/resources/devcontainer-Dockerfile" >> "$OUT_FILE"
+
+echo "Generated $OUT_FILE using root dir $ROOT_DIR" >&2
--- a/.devcontainer/scripts/restart-backend.sh
+++ b/.devcontainer/scripts/restart-backend.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+# Start (or restart) the NetAlertX Python backend under debugpy in background.
+# This script is invoked by the VS Code task "Restart GraphQL".
+# It exists to avoid complex inline command chains that were being mangled by the task runner.
+
+set -e
+
+LOG_DIR=/app/log
+APP_DIR=/app/server
+PY=python3
+PORT_DEBUG=5678
+
+# Kill any prior debug/run instances
+sudo killall python3 2>/dev/null || true
+sleep 2
+
+
+cd "$APP_DIR"
+
+# Launch using absolute module path for clarity; rely on cwd for local imports
+setsid nohup ${PY} -m debugpy --listen 0.0.0.0:${PORT_DEBUG} /app/server/__main__.py >/dev/null 2>&1 &
+PID=$!
+sleep 2
+
--- a/.devcontainer/scripts/run-tests.sh
+++ b/.devcontainer/scripts/run-tests.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# shellcheck shell=sh
+# Simple helper to run pytest inside the devcontainer with correct paths
+set -eu
+
+# Ensure we run from the workspace root
+cd /workspaces/NetAlertX
+
+# Make sure PYTHONPATH includes server and workspace
+export PYTHONPATH="/workspaces/NetAlertX:/workspaces/NetAlertX/server:/app:/app/server:${PYTHONPATH:-}"
+
+# Default to running the full test suite under /workspaces/NetAlertX/test
+pytest -q --maxfail=1 --disable-warnings test "$@"
--- a/.devcontainer/scripts/setup.sh
+++ b/.devcontainer/scripts/setup.sh
@@ -0,0 +1,192 @@
+#! /bin/bash
+# Runtime setup for devcontainer (executed after container starts).
+# Prefer building setup into resources/devcontainer-Dockerfile when possible.
+# Use this script for runtime-only adjustments (permissions, sockets, ownership,
+# and services managed without init) that are difficult at build time.
+id
+
+# Define variables (paths, ports, environment)
+
+export APP_DIR="/app"
+export APP_COMMAND="/workspaces/NetAlertX/.devcontainer/scripts/restart-backend.sh"
+export PHP_FPM_BIN="/usr/sbin/php-fpm83"
+export NGINX_BIN="/usr/sbin/nginx"
+export CROND_BIN="/usr/sbin/crond -f"
+
+
+export ALWAYS_FRESH_INSTALL=false
+export INSTALL_DIR=/app
+export APP_DATA_LOCATION=/app/config
+export APP_CONFIG_LOCATION=/app/config
+export LOGS_LOCATION=/app/logs
+export CONF_FILE="app.conf"
+export NGINX_CONF_FILE=netalertx.conf
+export DB_FILE="app.db"
+export FULL_FILEDB_PATH="${INSTALL_DIR}/db/${DB_FILE}"
+export NGINX_CONFIG_FILE="/etc/nginx/http.d/${NGINX_CONF_FILE}"
+export OUI_FILE="/usr/share/arp-scan/ieee-oui.txt" # Define the path to ieee-oui.txt and ieee-iab.txt
+export TZ=Europe/Paris
+export PORT=20211
+export SOURCE_DIR="/workspaces/NetAlertX"
+
+
+ 
+main() {
+    echo "=== NetAlertX Development Container Setup ==="
+    echo "Setting up ${SOURCE_DIR}..."
+    configure_source
+    
+    echo "--- Starting Development Services ---"
+    configure_php
+
+
+    start_services
+}
+
+# safe_link: create a symlink from source to target, removing existing target if necessary
+# bypassing the default behavior of symlinking the directory into the target directory if it is a directory
+safe_link() {
+    # usage: safe_link <source> <target>
+    local src="$1"
+    local dst="$2"
+
+    # Ensure parent directory exists
+    install -d -m 775 "$(dirname "$dst")" >/dev/null 2>&1 || true
+
+    # If target exists, remove it without dereferencing symlinks
+    if [ -L "$dst" ] || [ -e "$dst" ]; then
+        rm -rf "$dst"
+    fi
+
+    # Create link; -n prevents deref, -f replaces if somehow still exists
+    ln -sfn "$src" "$dst"
+}
+
+# Setup source directory
+configure_source() {
+    echo "[1/3] Configuring Source..."
+    echo "  -> Linking source to ${INSTALL_DIR}"
+    echo "Dev">${INSTALL_DIR}/.VERSION
+    safe_link ${SOURCE_DIR}/api ${INSTALL_DIR}/api
+    safe_link ${SOURCE_DIR}/back ${INSTALL_DIR}/back
+    safe_link "${SOURCE_DIR}/config"         "${INSTALL_DIR}/config"
+    safe_link "${SOURCE_DIR}/db"         "${INSTALL_DIR}/db"
+    if [ ! -f "${SOURCE_DIR}/config/app.conf" ]; then
+        cp ${SOURCE_DIR}/back/app.conf ${INSTALL_DIR}/config/
+        cp ${SOURCE_DIR}/back/app.db ${INSTALL_DIR}/db/
+    fi
+
+    safe_link "${SOURCE_DIR}/docs"       "${INSTALL_DIR}/docs"
+    safe_link "${SOURCE_DIR}/front"      "${INSTALL_DIR}/front"
+    safe_link "${SOURCE_DIR}/install"    "${INSTALL_DIR}/install"
+    safe_link "${SOURCE_DIR}/scripts"    "${INSTALL_DIR}/scripts"
+    safe_link "${SOURCE_DIR}/server"     "${INSTALL_DIR}/server"
+    safe_link "${SOURCE_DIR}/test"       "${INSTALL_DIR}/test"
+    safe_link "${SOURCE_DIR}/mkdocs.yml" "${INSTALL_DIR}/mkdocs.yml"
+
+    echo "  -> Copying static files to ${INSTALL_DIR}"
+    cp -R ${SOURCE_DIR}/CODE_OF_CONDUCT.md ${INSTALL_DIR}/
+    cp -R ${SOURCE_DIR}/dockerfiles ${INSTALL_DIR}/dockerfiles
+    sudo cp -na "${INSTALL_DIR}/back/${CONF_FILE}" "${INSTALL_DIR}/config/${CONF_FILE}"
+    sudo cp -na "${INSTALL_DIR}/back/${DB_FILE}" "${FULL_FILEDB_PATH}"
+    if [ -e "${INSTALL_DIR}/api/user_notifications.json" ]; then
+        echo "  -> Removing existing user_notifications.json"
+        sudo rm "${INSTALL_DIR}"/api/user_notifications.json
+    fi
+    
+    echo "  -> Setting ownership and permissions"
+    sudo find ${INSTALL_DIR}/ -type d -exec chmod 775 {} \;
+    sudo find ${INSTALL_DIR}/ -type f -exec chmod 664 {} \;
+    sudo date +%s > "${INSTALL_DIR}/front/buildtimestamp.txt"
+    sudo chmod 640 "${INSTALL_DIR}/config/${CONF_FILE}" || true
+
+    echo "  -> Setting up log directory"
+    sudo rm -Rf ${INSTALL_DIR}/log
+    install -d -o netalertx -g www-data -m 777 ${INSTALL_DIR}/log
+    install -d -o netalertx -g www-data -m 777 ${INSTALL_DIR}/log/plugins
+
+    echo "  -> Empty log"|tee ${INSTALL_DIR}/log/app.log \
+        ${INSTALL_DIR}/log/app_front.log \
+        ${INSTALL_DIR}/log/stdout.log
+    touch ${INSTALL_DIR}/log/stderr.log \
+    ${INSTALL_DIR}/log/execution_queue.log
+
+    date +%s > /app/front/buildtimestamp.txt
+
+    killall python &>/dev/null
+    sleep 1
+}
+
+#
+
+# start_services: start crond, PHP-FPM, nginx and the application
+start_services() {
+    echo "[3/3] Starting services..."
+
+    killall nohup &>/dev/null || true
+
+    killall php-fpm83 &>/dev/null || true
+    killall crond &>/dev/null || true
+    # Give the OS a moment to release the php-fpm socket
+    sleep 0.3
+    echo "      -> Starting CronD"
+    setsid nohup $CROND_BIN &>/dev/null &
+
+    echo "      -> Starting PHP-FPM"
+    setsid nohup $PHP_FPM_BIN &>/dev/null &
+
+    sudo killall nginx &>/dev/null || true
+    # Wait for the previous nginx processes to exit and for the port to free up
+    tries=0
+    while ss -ltn | grep -q ":${PORT}[[:space:]]" && [ $tries -lt 10 ]; do
+        echo "  -> Waiting for port ${PORT} to free..."
+        sleep 0.2
+        tries=$((tries+1))
+    done
+    sleep 0.2
+    echo "      -> Starting Nginx"
+    setsid nohup $NGINX_BIN &>/dev/null &
+    echo "      -> Starting Backend ${APP_DIR}/server..."
+    $APP_COMMAND 
+    sleep 2
+}
+
+# configure_php: configure PHP-FPM and enable dev debug options
+configure_php() {
+    echo "[2/3] Configuring PHP-FPM..."
+    sudo killall php-fpm83 &>/dev/null || true
+    install -d -o nginx -g www-data /run/php/ &>/dev/null
+    sudo sed -i "/^;pid/c\pid = /run/php/php8.3-fpm.pid" /etc/php83/php-fpm.conf
+    sudo sed -i 's|^listen = .*|listen = 127.0.0.1:9000|' /etc/php83/php-fpm.d/www.conf
+    sudo sed -i 's|fastcgi_pass .*|fastcgi_pass 127.0.0.1:9000;|' /etc/nginx/http.d/*.conf
+
+    #increase max child process count to 10
+    sudo sed -i -e 's/pm.max_children = 5/pm.max_children = 10/' /etc/php83/php-fpm.d/www.conf 
+
+    # find any line in php-fmp that starts with either ;error_log or error_log = and replace it with error_log = /app/log/app.php_errors.log
+    sudo sed -i '/^;*error_log\s*=/c\error_log = /app/log/app.php_errors.log' /etc/php83/php-fpm.conf
+    # If the line was not found, append it to the end of the file
+    if ! grep -q '^error_log\s*=' /etc/php83/php-fpm.conf; then
+        echo 'error_log = /app/log/app.php_errors.log' | sudo tee -a /etc/php83/php-fpm.conf
+    fi
+
+    sudo mkdir -p /etc/php83/conf.d
+    sudo cp /workspaces/NetAlertX/.devcontainer/resources/99-xdebug.ini /etc/php83/conf.d/99-xdebug.ini
+
+    sudo rm -R /var/log/php83 &>/dev/null || true
+    install -d -o netalertx -g www-data -m 755 var/log/php83;
+
+    sudo chmod 644 /etc/php83/conf.d/99-xdebug.ini || true
+
+}
+
+# (duplicate start_services removed)
+
+
+
+echo "$(git rev-parse --short=8 HEAD)">/app/.VERSION
+# Run the main function
+main
+
+
+
--- a/.devcontainer/scripts/stream-logs.sh
+++ b/.devcontainer/scripts/stream-logs.sh
@@ -0,0 +1,40 @@
+#!/bin/sh
+# Stream NetAlertX logs to stdout so the Dev Containers output channel shows them.
+# This script waits briefly for the files to appear and then tails them with -F.
+
+LOG_FILES="/app/log/app.log  /app/log/db_is_locked.log /app/log/execution_queue.log /app/log/app_front.log /app/log/app.php_errors.log /app/log/IP_changes.log /app/stderr.log /app/stdout.log"
+
+wait_for_files() {
+  # Wait up to ~10s for at least one of the files to exist
+  attempts=0
+  while [ $attempts -lt 20 ]; do
+    for f in $LOG_FILES; do
+      if [ -f "$f" ]; then
+        return 0
+      fi
+    done
+    attempts=$((attempts+1))
+    sleep 0.5
+  done
+  return 1
+}
+
+if wait_for_files; then
+  echo "Starting log stream for:"
+  for f in $LOG_FILES; do
+    [ -f "$f" ] && echo "  $f"
+  done
+
+  # Use tail -F where available. If tail -F isn't supported, tail -f is used as fallback.
+  # Some minimal images may have busybox tail without -F; this handles both.
+  if tail --version >/dev/null 2>&1; then
+    # GNU tail supports -F
+    tail -n +1 -F $LOG_FILES
+  else
+    # Fallback to -f for busybox; will exit if files rotate or do not exist initially
+    tail -n +1 -f $LOG_FILES
+  fi
+else
+  echo "No log files appeared after wait; exiting stream script."
+  exit 0
+fi
--- a/.devcontainer/xdebug-trigger.ini
+++ b/.devcontainer/xdebug-trigger.ini
@@ -0,0 +1,11 @@
+zend_extension=xdebug.so
+xdebug.mode=debug
+xdebug.start_with_request=trigger
+xdebug.trigger_value=VSCODE
+xdebug.client_host=host.docker.internal
+xdebug.client_port=9003
+xdebug.log=/var/log/xdebug.log
+xdebug.log_level=7
+xdebug.idekey=VSCODE
+xdebug.discover_client_host=true
+xdebug.max_nesting_level=512
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -0,0 +1,48 @@
+This is NetAlertX — network monitoring & alerting.
+
+Purpose: Guide AI assistants to follow NetAlertX architecture, conventions, and safety practices. Be concise, opinionated, and prefer existing helpers/settings over new code or hardcoded values.
+
+## Architecture (what runs where)
+- Backend (Python): main loop + GraphQL/REST endpoints orchestrate scans, plugins, workflows, notifications, and JSON export.
+	- Key: `server/__main__.py`, `server/plugin.py`, `server/initialise.py`, `server/api_server/api_server_start.py`
+- Data (SQLite): persistent state in `db/app.db`; helpers in `server/database.py` and `server/db/*`.
+- Frontend (Nginx + PHP + JS): UI reads JSON, triggers execution queue events.
+	- Key: `front/`, `front/js/common.js`, `front/php/server/*.php`
+- Plugins (Python): acquisition/enrichment/publishers under `front/plugins/*` with `config.json` manifests.
+- Messaging/Workflows: `server/messaging/*`, `server/workflows/*`
+- API JSON Cache for UI: generated under `api/*.json`
+
+Backend loop phases (see `server/__main__.py` and `server/plugin.py`): `once`, `schedule`, `always_after_scan`, `before_name_updates`, `on_new_device`, `on_notification`, plus ad‑hoc `run` via execution queue. Plugins execute as scripts that write result logs for ingestion.
+
+## Plugin patterns that matter
+- Manifest lives at `front/plugins/<code_name>/config.json`; `code_name` == folder, `unique_prefix` drives settings and filenames (e.g., `ARPSCAN`).
+- Control via settings: `<PREF>_RUN` (phase), `<PREF>_RUN_SCHD` (cron-like), `<PREF>_CMD` (script path), `<PREF>_RUN_TIMEOUT`, `<PREF>_WATCH` (diff columns).
+- Data contract: scripts write `/app/log/plugins/last_result.<PREF>.log` (pipe‑delimited: 9 required cols + optional 4). Use `front/plugins/plugin_helper.py`’s `Plugin_Objects` to sanitize text and normalize MACs, then `write_result_file()`.
+- Device import: define `database_column_definitions` when creating/updating devices; watched fields trigger notifications.
+
+## API/Endpoints quick map
+- Flask app: `server/api_server/api_server_start.py` exposes routes like `/device/<mac>`, `/devices`, `/devices/export/{csv,json}`, `/devices/import`, `/devices/totals`, `/devices/by-status`, plus `nettools`, `events`, `sessions`, `dbquery`, `metrics`, `sync`.
+- Authorization: all routes expect header `Authorization: Bearer <API_TOKEN>` via `get_setting_value('API_TOKEN')`.
+
+## Conventions & helpers to reuse
+- Settings: add/modify via `ccd()` in `server/initialise.py` or per‑plugin manifest. Never hardcode ports or secrets; use `get_setting_value()`.
+- Logging: use `logger.mylog(level, [message])`; levels: none/minimal/verbose/debug/trace.
+- Time/MAC/strings: `helper.py` (`timeNowTZ`, `normalize_mac`, sanitizers). Validate MACs before DB writes.
+- DB helpers: prefer `server/db/db_helper.py` functions (e.g., `get_table_json`, device condition helpers) over raw SQL in new paths.
+
+## Dev workflow (devcontainer)
+- Services: use tasks to (re)start backend and nginx/PHP-FPM. Backend runs with debugpy on 5678; attach a Python debugger if needed.
+- Run a plugin manually: `python3 front/plugins/<code_name>/script.py` (ensure `sys.path` includes `/app/front/plugins` and `/app/server` like the template).
+- Testing: pytest available via Alpine packages. Tests live in `test/`; app code is under `server/`. PYTHONPATH is preconfigured to include workspace and `/opt/venv` site‑packages.
+
+## What “done right” looks like
+- When adding a plugin, start from `front/plugins/__template`, implement with `plugin_helper`, define manifest settings, and wire phase via `<PREF>_RUN`. Verify logs in `/app/log/plugins/` and data in `api/*.json`.
+- When introducing new config, define it once (core `ccd()` or plugin manifest) and read it via helpers everywhere.
+- When exposing new server functionality, add endpoints in `server/api_server/*` and keep authorization consistent; update UI by reading/writing JSON cache rather than bypassing the pipeline.
+
+## Useful references
+- Docs: `docs/PLUGINS_DEV.md`, `docs/SETTINGS_SYSTEM.md`, `docs/API_*.md`, `docs/DEBUG_*.md`
+- Logs: backend `/app/log/app.log`, plugin logs under `/app/log/plugins/`, nginx/php logs under `/var/log/*`
+
+Assistant expectations
+- Reference concrete files/paths. Use existing helpers/settings. Keep changes idempotent and safe. Offer a quick validation step (log line, API hit, or JSON export) for anything you add.
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -0,0 +1,34 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Backend Debug: Attach",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    // Map workspace root to /app for PHP and other resources, plus explicit server mapping for Python.
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/app"
+                },
+                {
+                    "localRoot": "${workspaceFolder}/server",
+                    "remoteRoot": "/app/server"
+                }
+            ]
+        },
+        {
+            "name": "PHP Frontend Xdebug: Listen",
+            "type": "php",
+            "request": "launch",
+            "port": 9003,
+            "pathMappings": {
+                "/app": "${workspaceFolder}"
+            }
+        }
+    ]
+}
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -0,0 +1,13 @@
+{
+    "terminal.integrated.suggest.enabled": true,
+    // Use pytest and look under the test/ folder
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestArgs": [
+        "test"
+    ],
+    // Ensure VS Code uses the devcontainer virtualenv
+    "python.defaultInterpreterPath": "/opt/venv/bin/python",
+    // Let the Python extension invoke pytest via the interpreter; avoid hardcoded paths
+    // Removed python.testing.pytestPath and legacy pytest.command overrides
+}
--- a/.vscode/tasks.json
+++ b/.vscode/tasks.json
@@ -0,0 +1,94 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "Generate Dockerfile",
+      "type": "shell",
+      "command": "${workspaceFolder:NetAlertX}/.devcontainer/scripts/generate-dockerfile.sh",
+      "presentation": {
+        "echo": true,
+        "reveal": "always",
+        "panel": "shared",
+        "showReuseMessage": false
+      },
+      "problemMatcher": [],
+      "group": {
+        "kind": "build",
+        "isDefault": false
+      },
+      "options": {
+        "cwd": "${workspaceFolder:NetAlertX}"
+      },
+      "icon": {
+        "id": "tools",
+        "color": "terminal.ansiYellow"
+      }
+    },
+    {
+      "label": "Re-Run Startup Script",
+      "type": "shell",
+      "command": "${workspaceFolder:NetAlertX}/.devcontainer/scripts/setup.sh",
+      "presentation": {
+        "echo": true,
+        "reveal": "always",
+        "panel": "shared",
+        "showReuseMessage": false
+      },
+      "problemMatcher": [],
+      "icon": {
+        "id": "beaker",
+        "color": "terminal.ansiBlue"
+      }
+    },
+    {
+      "label": "Start Backend (Python)",
+      "type": "shell",
+      "command": "/workspaces/NetAlertX/.devcontainer/scripts/restart-backend.sh",
+      "presentation": {
+        "echo": true,
+        "reveal": "always",
+        "panel": "shared",
+        "showReuseMessage": false,
+        "clear": false
+      },
+      "problemMatcher": [],
+      "icon": {
+        "id": "debug-restart",
+        "color": "terminal.ansiGreen"
+      }
+    },
+    {
+      "label": "Start Frontend (nginx and PHP-FPM)",
+      "type": "shell",
+      "command": "killall php-fpm83 nginx 2>/dev/null || true; sleep 1; php-fpm83 & nginx",
+      "presentation": {
+        "echo": true,
+        "reveal": "always",
+        "panel": "shared",
+        "showReuseMessage": false,
+        "clear": false
+      },
+      "problemMatcher": [],
+      "icon": {
+        "id": "debug-restart",
+        "color": "terminal.ansiGreen"
+      }
+    },
+    {
+      "label": "Stop Frontend & Backend Services",
+      "type": "shell",
+      "command": "pkill -f 'php-fpm83|nginx|crond|python3' || true",
+      "presentation": {
+        "echo": true,
+        "reveal": "always",
+        "panel": "shared",
+        "showReuseMessage": false
+      },
+      "problemMatcher": [],
+      "icon": {
+        "id": "debug-stop",
+        "color": "terminal.ansiRed"
+      }
+    }
+  ]
+}
--- a/front/.gitignore
+++ b/front/.gitignore
@@ -0,0 +1 @@
+buildtimestamp.txt
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -0,0 +1,5 @@
+[tool.pytest.ini_options]
+python_classes = ["Test", "Describe"]
+python_functions = ["test_", "it_", "and_", "but_", "they_"]
+python_files = ["test_*.py",]
+testpaths = ["test",]