diff --git a/.coverage b/.coverage new file mode 100644 index 00000000..96d3d1ac Binary files /dev/null and b/.coverage differ diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100755 index 00000000..d735514c --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,112 @@ +# DO NOT MODIFY THIS FILE DIRECTLY. IT IS AUTO-GENERATED BY .devcontainer/scripts/generate-dockerfile.sh + +# ---/Dockerfile--- +FROM alpine:3.22 AS builder + +ARG INSTALL_DIR=/app + +ENV PYTHONUNBUFFERED=1 + +# Install build dependencies +RUN apk add --no-cache bash shadow python3 python3-dev gcc musl-dev libffi-dev openssl-dev git \ + && python -m venv /opt/venv + +# Enable venv +ENV PATH="/opt/venv/bin:$PATH" + + +RUN pip install openwrt-luci-rpc asusrouter asyncio aiohttp graphene flask flask-cors unifi-sm-api tplink-omada-client wakeonlan pycryptodome requests paho-mqtt scapy cron-converter pytz json2table dhcp-leases pyunifi speedtest-cli chardet python-nmap dnspython librouteros yattag git+https://github.com/foreign-sub/aiofreepybox.git + +# Append Iliadbox certificate to aiofreepybox + +# second stage +FROM alpine:3.22 AS runner + +ARG INSTALL_DIR=/app + +COPY --from=builder /opt/venv /opt/venv +COPY --from=builder /usr/sbin/usermod /usr/sbin/groupmod /usr/sbin/ + +# Enable venv +ENV PATH="/opt/venv/bin:$PATH" + +# default port and listen address +ENV PORT=20211 LISTEN_ADDR=0.0.0.0 + +# needed for s6-overlay +ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 + +# ❗ IMPORTANT - if you modify this file modify the /install/install_dependecies.sh file as well ❗ + +RUN apk update --no-cache \ + && apk add --no-cache bash libbsd zip lsblk gettext-envsubst sudo mtr tzdata s6-overlay \ + && apk add --no-cache curl arp-scan iproute2 iproute2-ss nmap nmap-scripts traceroute nbtscan avahi avahi-tools openrc dbus net-tools net-snmp-tools bind-tools awake ca-certificates \ + && apk add --no-cache sqlite php83 php83-fpm php83-cgi php83-curl php83-sqlite3 php83-session \ + && apk add --no-cache python3 nginx \ + && ln -s /usr/bin/awake /usr/bin/wakeonlan \ + && rm -f /etc/nginx/http.d/default.conf + + +# Add crontab file +COPY --chmod=600 --chown=root:root install/crontab /etc/crontabs/root + +# Start all required services + +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=2 \ + CMD curl -sf -o /dev/null ${LISTEN_ADDR}:${PORT}/php/server/query_json.php?file=app_state.json + +ENTRYPOINT ["/init"] + +# ---/resources/devcontainer-Dockerfile--- + +# Devcontainer build stage (do not build directly) +# This file is combined with the root /Dockerfile by +# .devcontainer/scripts/generate-dockerfile.sh +# The generator appends this stage to produce .devcontainer/Dockerfile. +# Prefer to place dev-only setup here; use setup.sh only for runtime fixes. + +FROM runner AS devcontainer +ENV INSTALL_DIR=/app +ENV PYTHONPATH=/workspaces/NetAlertX/test:/workspaces/NetAlertX/server:/app:/app/server:/opt/venv/lib/python3.12/site-packages + +# Install common tools, create user, and set up sudo +RUN apk add --no-cache git nano vim jq php83-pecl-xdebug py3-pip nodejs sudo gpgconf pytest pytest-cov && \ + adduser -D -s /bin/sh netalertx && \ + addgroup netalertx nginx && \ + addgroup netalertx www-data && \ + echo "netalertx ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-netalertx && \ + chmod 440 /etc/sudoers.d/90-netalertx +# Install debugpy in the virtualenv if present, otherwise into system python3 +RUN /bin/sh -c '(/opt/venv/bin/python3 -m pip install --no-cache-dir debugpy) || (python3 -m pip install --no-cache-dir debugpy) || true' +# setup nginx +COPY .devcontainer/resources/netalertx-devcontainer.conf /etc/nginx/http.d/netalert-frontend.conf +RUN set -e; \ + chown netalertx:nginx /etc/nginx/http.d/netalert-frontend.conf; \ + install -d -o netalertx -g www-data -m 775 /app; \ + install -d -o netalertx -g www-data -m 755 /run/nginx; \ + install -d -o netalertx -g www-data -m 755 /var/lib/nginx/logs; \ + rm -f /var/lib/nginx/logs/* || true; \ + for f in error access; do : > /var/lib/nginx/logs/$f.log; done; \ + install -d -o netalertx -g www-data -m 777 /run/php; \ + install -d -o netalertx -g www-data -m 775 /var/log/php; \ + chown -R netalertx:www-data /etc/nginx/http.d; \ + chmod -R 775 /etc/nginx/http.d; \ + chown -R netalertx:www-data /var/lib/nginx; \ + chmod -R 755 /var/lib/nginx && \ + chown -R netalertx:www-data /var/log/nginx/ && \ + sed -i '/^user /d' /etc/nginx/nginx.conf; \ + sed -i 's|^error_log .*|error_log /dev/stderr warn;|' /etc/nginx/nginx.conf; \ + sed -i 's|^access_log .*|access_log /dev/stdout main;|' /etc/nginx/nginx.conf; \ + sed -i 's|error_log .*|error_log /dev/stderr warn;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \ + sed -i 's|access_log .*|access_log /dev/stdout main;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \ + mkdir -p /run/openrc; \ + chown netalertx:nginx /run/openrc/; \ + rm -Rf /run/openrc/*; + +# setup pytest +RUN sudo /opt/venv/bin/python -m pip install -U pytest pytest-cov + +WORKDIR /workspaces/NetAlertX + + +ENTRYPOINT ["/bin/sh","-c","sleep infinity"] \ No newline at end of file diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000..9fa909e7 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,30 @@ +# NetAlertX Devcontainer Notes + +This devcontainer replicates the production container as closely as practical, with a few development-oriented differences. + +Key behavior +- No init process: Services are managed by shell scripts using killall, setsid, and nohup. Startup and restarts are script-driven rather than supervised by an init system. +- Autogenerated Dockerfile: The effective devcontainer Dockerfile is generated on demand by `.devcontainer/scripts/generate-dockerfile.sh`. It combines the root `Dockerfile` (with certain COPY instructions removed) and an extra "devcontainer" stage from `.devcontainer/resources/devcontainer-Dockerfile`. When you change the resource Dockerfile, re-run the generator to refresh `.devcontainer/Dockerfile`. +- Where to put setup: Prefer baking setup into `.devcontainer/resources/devcontainer-Dockerfile`. Use `.devcontainer/scripts/setup.sh` only for steps that must happen at container start (e.g., cleaning up nginx/php ownership, creating directories, touching runtime files) or depend on runtime paths. + +Debugging (F5) +The Frontend and backend run in debug mode always. You can attach your debugger at any time. +- Python Backend Debug: Attach - The backend runs with a debugger on port 5678. Set breakpoints in the code and press F5 to begin triggering them. +- PHP Frontend (XDebug) Xdebug listens on 9003. Start listening and use an Xdebug extension in your browser to debug PHP. + +Common workflows (F1->Tasks: Run Task) +- Regenerate the devcontainer Dockerfile: Run the VS Code task "Generate Dockerfile" or execute `.devcontainer/scripts/generate-dockerfile.sh`. The result is `.devcontainer/Dockerfile`. +- Re-run startup provisioning: Use the task "Re-Run Startup Script" to execute `.devcontainer/scripts/setup.sh` in the container. +- Start services: + - Backend (GraphQL/Flask): `.devcontainer/scripts/restart-backend.sh` starts it under debugpy and logs to `/app/log/app.log` + - Frontend (nginx + PHP-FPM): Started via setup.sh; can be restarted by the task "Start Frontend (nginx and PHP-FPM)". + +Testing +- pytest is installed via Alpine packages (py3-pytest, py3-pytest-cov). +- PYTHONPATH includes workspace and venv site-packages so tests can import `server/*` modules and third-party libs. +- Run tests via VS Code Pytest Runner or `pytest -q` from the workspace root. + +Conventions +- Don’t edit `.devcontainer/Dockerfile` directly; edit `.devcontainer/resources/devcontainer-Dockerfile` and regenerate. +- Keep setup in the resource Dockerfile when possible; reserve `setup.sh` for runtime fixes. +- Avoid hardcoding ports/secrets; prefer existing settings and helpers (see `.github/copilot-instructions.md`). \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100755 index 00000000..f9f6440e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,75 @@ +{ + "name": "NetAlertX DevContainer", + "remoteUser": "netalertx", + "build": { + "dockerfile": "Dockerfile", + "context": "..", + "target": "devcontainer" + }, + "workspaceFolder": "/workspaces/NetAlertX", + + "runArgs": [ + "--privileged", + "--cap-add=NET_ADMIN", + "--cap-add=NET_RAW", + // Ensure containers can resolve host.docker.internal to the host (required on Linux) + "--add-host=host.docker.internal:host-gateway" + ], + + "postStartCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/setup.sh", + + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker", + "felixfbecker.php-debug", + "bmewburn.vscode-intelephense-client", + "xdebug.php-debug", + "ms-python.vscode-pylance", + "pamaron.pytest-runner", + "coderabbit.coderabbit-vscode", + "ms-python.black-formatter" + ] + , + "settings": { + "terminal.integrated.cwd": "${containerWorkspaceFolder}", + // Python testing configuration + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": [ + "test" + ], + // Make sure we discover tests and import server correctly + "python.analysis.extraPaths": [ + "/workspaces/NetAlertX", + "/workspaces/NetAlertX/server", + "/app", + "/app/server" + ] + } + } + }, + "forwardPorts": [5678, 9000, 9003, 20211, 20212], + + "portsAttributes": { + "20211": { + "label": "Frontend:Nginx+PHP" + }, + "20212": { + "label": "Backend:GraphQL" + }, + "9003": { + "label": "PHP Debug:Xdebug" + }, + "9000": { + "label": "PHP-FPM:FastCGI" + }, + "5678": { + "label": "Python Debug:debugpy" + } + }, + + // Optional: ensures compose services are stopped when you close the window + "shutdownAction": "stopContainer" +} \ No newline at end of file diff --git a/.devcontainer/resources/99-xdebug.ini b/.devcontainer/resources/99-xdebug.ini new file mode 100644 index 00000000..37452d58 --- /dev/null +++ b/.devcontainer/resources/99-xdebug.ini @@ -0,0 +1,8 @@ +zend_extension="xdebug.so" +[xdebug] +xdebug.mode=develop,debug +xdebug.log_level=0 +xdebug.client_host=host.docker.internal +xdebug.client_port=9003 +xdebug.start_with_request=yes +xdebug.discover_client_host=1 diff --git a/.devcontainer/resources/devcontainer-Dockerfile b/.devcontainer/resources/devcontainer-Dockerfile new file mode 100644 index 00000000..88ef4ece --- /dev/null +++ b/.devcontainer/resources/devcontainer-Dockerfile @@ -0,0 +1,51 @@ +# Devcontainer build stage (do not build directly) +# This file is combined with the root /Dockerfile by +# .devcontainer/scripts/generate-dockerfile.sh +# The generator appends this stage to produce .devcontainer/Dockerfile. +# Prefer to place dev-only setup here; use setup.sh only for runtime fixes. + +FROM runner AS devcontainer +ENV INSTALL_DIR=/app +ENV PYTHONPATH=/workspaces/NetAlertX/test:/workspaces/NetAlertX/server:/app:/app/server:/opt/venv/lib/python3.12/site-packages + +# Install common tools, create user, and set up sudo +RUN apk add --no-cache git nano vim jq php83-pecl-xdebug py3-pip nodejs sudo gpgconf pytest pytest-cov && \ + adduser -D -s /bin/sh netalertx && \ + addgroup netalertx nginx && \ + addgroup netalertx www-data && \ + echo "netalertx ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-netalertx && \ + chmod 440 /etc/sudoers.d/90-netalertx +# Install debugpy in the virtualenv if present, otherwise into system python3 +RUN /bin/sh -c '(/opt/venv/bin/python3 -m pip install --no-cache-dir debugpy) || (python3 -m pip install --no-cache-dir debugpy) || true' +# setup nginx +COPY .devcontainer/resources/netalertx-devcontainer.conf /etc/nginx/http.d/netalert-frontend.conf +RUN set -e; \ + chown netalertx:nginx /etc/nginx/http.d/netalert-frontend.conf; \ + install -d -o netalertx -g www-data -m 775 /app; \ + install -d -o netalertx -g www-data -m 755 /run/nginx; \ + install -d -o netalertx -g www-data -m 755 /var/lib/nginx/logs; \ + rm -f /var/lib/nginx/logs/* || true; \ + for f in error access; do : > /var/lib/nginx/logs/$f.log; done; \ + install -d -o netalertx -g www-data -m 777 /run/php; \ + install -d -o netalertx -g www-data -m 775 /var/log/php; \ + chown -R netalertx:www-data /etc/nginx/http.d; \ + chmod -R 775 /etc/nginx/http.d; \ + chown -R netalertx:www-data /var/lib/nginx; \ + chmod -R 755 /var/lib/nginx && \ + chown -R netalertx:www-data /var/log/nginx/ && \ + sed -i '/^user /d' /etc/nginx/nginx.conf; \ + sed -i 's|^error_log .*|error_log /dev/stderr warn;|' /etc/nginx/nginx.conf; \ + sed -i 's|^access_log .*|access_log /dev/stdout main;|' /etc/nginx/nginx.conf; \ + sed -i 's|error_log .*|error_log /dev/stderr warn;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \ + sed -i 's|access_log .*|access_log /dev/stdout main;|g' /etc/nginx/http.d/*.conf 2>/dev/null || true; \ + mkdir -p /run/openrc; \ + chown netalertx:nginx /run/openrc/; \ + rm -Rf /run/openrc/*; + +# setup pytest +RUN sudo /opt/venv/bin/python -m pip install -U pytest pytest-cov + +WORKDIR /workspaces/NetAlertX + + +ENTRYPOINT ["/bin/sh","-c","sleep infinity"] \ No newline at end of file diff --git a/.devcontainer/resources/netalertx-devcontainer.conf b/.devcontainer/resources/netalertx-devcontainer.conf new file mode 100644 index 00000000..be8f1cca --- /dev/null +++ b/.devcontainer/resources/netalertx-devcontainer.conf @@ -0,0 +1,26 @@ +log_format netalertx '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; +access_log /var/log/nginx/access.log netalertx flush=1s; +error_log /var/log/nginx/error.log warn; + +server { + listen 20211 default_server; + root /app/front; + index index.php; + + add_header X-Forwarded-Prefix "/netalertx" always; + proxy_set_header X-Forwarded-Prefix "/netalertx"; + + location ~* \.php$ { + add_header Cache-Control "no-store"; + fastcgi_pass 127.0.0.1:9000; + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; + fastcgi_param SCRIPT_NAME $fastcgi_script_name; + fastcgi_param PHP_VALUE "xdebug.remote_enable=1"; + fastcgi_connect_timeout 75; + fastcgi_send_timeout 600; + fastcgi_read_timeout 600; + } +} diff --git a/.devcontainer/scripts/generate-dockerfile.sh b/.devcontainer/scripts/generate-dockerfile.sh new file mode 100755 index 00000000..d97cefd9 --- /dev/null +++ b/.devcontainer/scripts/generate-dockerfile.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Generator for .devcontainer/Dockerfile +# Combines the root /Dockerfile (with some COPY lines removed) and +# the dev-only stage in .devcontainer/resources/devcontainer-Dockerfile. +# Run this script after modifying the resource Dockerfile to refresh +# the final .devcontainer/Dockerfile used by the devcontainer. + +# Make a copy of the original Dockerfile to the .devcontainer folder +# but remove the COPY . ${INSTALL_DIR}/ command from it. This avoids +# overwriting /app (which uses symlinks to the workspace) and preserves +# debugging capabilities inside the devcontainer. + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +DEVCONTAINER_DIR="${SCRIPT_DIR%/scripts}" +ROOT_DIR="${DEVCONTAINER_DIR%/.devcontainer}" + +OUT_FILE="${DEVCONTAINER_DIR}/Dockerfile" + +echo "# DO NOT MODIFY THIS FILE DIRECTLY. IT IS AUTO-GENERATED BY .devcontainer/scripts/generate-dockerfile.sh" > "$OUT_FILE" +echo "" >> "$OUT_FILE" +echo "# ---/Dockerfile---" >> "$OUT_FILE" + +sed '/${INSTALL_DIR}/d' "${ROOT_DIR}/Dockerfile" >> "$OUT_FILE" + +# sed the line https://github.com/foreign-sub/aiofreepybox.git \\ to remove trailing backslash +sed -i '/aiofreepybox.git/ s/ \\$//' "$OUT_FILE" + +# don't cat the file, just copy it in because it doesn't exist at build time +sed -i 's|^ RUN cat ${INSTALL_DIR}/install/freebox_certificate.pem >> /opt/venv/lib/python3.12/site-packages/aiofreepybox/freebox_certificates.pem$| COPY install/freebox_certificate.pem /opt/venv/lib/python3.12/site-packages/aiofreepybox/freebox_certificates.pem |' "$OUT_FILE" + +echo "" >> "$OUT_FILE" +echo "# ---/resources/devcontainer-Dockerfile---" >> "$OUT_FILE" +echo "" >> "$OUT_FILE" + +cat "${DEVCONTAINER_DIR}/resources/devcontainer-Dockerfile" >> "$OUT_FILE" + +echo "Generated $OUT_FILE using root dir $ROOT_DIR" >&2 diff --git a/.devcontainer/scripts/restart-backend.sh b/.devcontainer/scripts/restart-backend.sh new file mode 100755 index 00000000..3416d561 --- /dev/null +++ b/.devcontainer/scripts/restart-backend.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Start (or restart) the NetAlertX Python backend under debugpy in background. +# This script is invoked by the VS Code task "Restart GraphQL". +# It exists to avoid complex inline command chains that were being mangled by the task runner. + +set -e + +LOG_DIR=/app/log +APP_DIR=/app/server +PY=python3 +PORT_DEBUG=5678 + +# Kill any prior debug/run instances +sudo killall python3 2>/dev/null || true +sleep 2 + + +cd "$APP_DIR" + +# Launch using absolute module path for clarity; rely on cwd for local imports +setsid nohup ${PY} -m debugpy --listen 0.0.0.0:${PORT_DEBUG} /app/server/__main__.py >/dev/null 2>&1 & +PID=$! +sleep 2 + diff --git a/.devcontainer/scripts/run-tests.sh b/.devcontainer/scripts/run-tests.sh new file mode 100755 index 00000000..80eaf013 --- /dev/null +++ b/.devcontainer/scripts/run-tests.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# shellcheck shell=sh +# Simple helper to run pytest inside the devcontainer with correct paths +set -eu + +# Ensure we run from the workspace root +cd /workspaces/NetAlertX + +# Make sure PYTHONPATH includes server and workspace +export PYTHONPATH="/workspaces/NetAlertX:/workspaces/NetAlertX/server:/app:/app/server:${PYTHONPATH:-}" + +# Default to running the full test suite under /workspaces/NetAlertX/test +pytest -q --maxfail=1 --disable-warnings test "$@" diff --git a/.devcontainer/scripts/setup.sh b/.devcontainer/scripts/setup.sh new file mode 100755 index 00000000..efba3270 --- /dev/null +++ b/.devcontainer/scripts/setup.sh @@ -0,0 +1,192 @@ +#! /bin/bash +# Runtime setup for devcontainer (executed after container starts). +# Prefer building setup into resources/devcontainer-Dockerfile when possible. +# Use this script for runtime-only adjustments (permissions, sockets, ownership, +# and services managed without init) that are difficult at build time. +id + +# Define variables (paths, ports, environment) + +export APP_DIR="/app" +export APP_COMMAND="/workspaces/NetAlertX/.devcontainer/scripts/restart-backend.sh" +export PHP_FPM_BIN="/usr/sbin/php-fpm83" +export NGINX_BIN="/usr/sbin/nginx" +export CROND_BIN="/usr/sbin/crond -f" + + +export ALWAYS_FRESH_INSTALL=false +export INSTALL_DIR=/app +export APP_DATA_LOCATION=/app/config +export APP_CONFIG_LOCATION=/app/config +export LOGS_LOCATION=/app/logs +export CONF_FILE="app.conf" +export NGINX_CONF_FILE=netalertx.conf +export DB_FILE="app.db" +export FULL_FILEDB_PATH="${INSTALL_DIR}/db/${DB_FILE}" +export NGINX_CONFIG_FILE="/etc/nginx/http.d/${NGINX_CONF_FILE}" +export OUI_FILE="/usr/share/arp-scan/ieee-oui.txt" # Define the path to ieee-oui.txt and ieee-iab.txt +export TZ=Europe/Paris +export PORT=20211 +export SOURCE_DIR="/workspaces/NetAlertX" + + + +main() { + echo "=== NetAlertX Development Container Setup ===" + echo "Setting up ${SOURCE_DIR}..." + configure_source + + echo "--- Starting Development Services ---" + configure_php + + + start_services +} + +# safe_link: create a symlink from source to target, removing existing target if necessary +# bypassing the default behavior of symlinking the directory into the target directory if it is a directory +safe_link() { + # usage: safe_link + local src="$1" + local dst="$2" + + # Ensure parent directory exists + install -d -m 775 "$(dirname "$dst")" >/dev/null 2>&1 || true + + # If target exists, remove it without dereferencing symlinks + if [ -L "$dst" ] || [ -e "$dst" ]; then + rm -rf "$dst" + fi + + # Create link; -n prevents deref, -f replaces if somehow still exists + ln -sfn "$src" "$dst" +} + +# Setup source directory +configure_source() { + echo "[1/3] Configuring Source..." + echo " -> Linking source to ${INSTALL_DIR}" + echo "Dev">${INSTALL_DIR}/.VERSION + safe_link ${SOURCE_DIR}/api ${INSTALL_DIR}/api + safe_link ${SOURCE_DIR}/back ${INSTALL_DIR}/back + safe_link "${SOURCE_DIR}/config" "${INSTALL_DIR}/config" + safe_link "${SOURCE_DIR}/db" "${INSTALL_DIR}/db" + if [ ! -f "${SOURCE_DIR}/config/app.conf" ]; then + cp ${SOURCE_DIR}/back/app.conf ${INSTALL_DIR}/config/ + cp ${SOURCE_DIR}/back/app.db ${INSTALL_DIR}/db/ + fi + + safe_link "${SOURCE_DIR}/docs" "${INSTALL_DIR}/docs" + safe_link "${SOURCE_DIR}/front" "${INSTALL_DIR}/front" + safe_link "${SOURCE_DIR}/install" "${INSTALL_DIR}/install" + safe_link "${SOURCE_DIR}/scripts" "${INSTALL_DIR}/scripts" + safe_link "${SOURCE_DIR}/server" "${INSTALL_DIR}/server" + safe_link "${SOURCE_DIR}/test" "${INSTALL_DIR}/test" + safe_link "${SOURCE_DIR}/mkdocs.yml" "${INSTALL_DIR}/mkdocs.yml" + + echo " -> Copying static files to ${INSTALL_DIR}" + cp -R ${SOURCE_DIR}/CODE_OF_CONDUCT.md ${INSTALL_DIR}/ + cp -R ${SOURCE_DIR}/dockerfiles ${INSTALL_DIR}/dockerfiles + sudo cp -na "${INSTALL_DIR}/back/${CONF_FILE}" "${INSTALL_DIR}/config/${CONF_FILE}" + sudo cp -na "${INSTALL_DIR}/back/${DB_FILE}" "${FULL_FILEDB_PATH}" + if [ -e "${INSTALL_DIR}/api/user_notifications.json" ]; then + echo " -> Removing existing user_notifications.json" + sudo rm "${INSTALL_DIR}"/api/user_notifications.json + fi + + echo " -> Setting ownership and permissions" + sudo find ${INSTALL_DIR}/ -type d -exec chmod 775 {} \; + sudo find ${INSTALL_DIR}/ -type f -exec chmod 664 {} \; + sudo date +%s > "${INSTALL_DIR}/front/buildtimestamp.txt" + sudo chmod 640 "${INSTALL_DIR}/config/${CONF_FILE}" || true + + echo " -> Setting up log directory" + sudo rm -Rf ${INSTALL_DIR}/log + install -d -o netalertx -g www-data -m 777 ${INSTALL_DIR}/log + install -d -o netalertx -g www-data -m 777 ${INSTALL_DIR}/log/plugins + + echo " -> Empty log"|tee ${INSTALL_DIR}/log/app.log \ + ${INSTALL_DIR}/log/app_front.log \ + ${INSTALL_DIR}/log/stdout.log + touch ${INSTALL_DIR}/log/stderr.log \ + ${INSTALL_DIR}/log/execution_queue.log + + date +%s > /app/front/buildtimestamp.txt + + killall python &>/dev/null + sleep 1 +} + +# + +# start_services: start crond, PHP-FPM, nginx and the application +start_services() { + echo "[3/3] Starting services..." + + killall nohup &>/dev/null || true + + killall php-fpm83 &>/dev/null || true + killall crond &>/dev/null || true + # Give the OS a moment to release the php-fpm socket + sleep 0.3 + echo " -> Starting CronD" + setsid nohup $CROND_BIN &>/dev/null & + + echo " -> Starting PHP-FPM" + setsid nohup $PHP_FPM_BIN &>/dev/null & + + sudo killall nginx &>/dev/null || true + # Wait for the previous nginx processes to exit and for the port to free up + tries=0 + while ss -ltn | grep -q ":${PORT}[[:space:]]" && [ $tries -lt 10 ]; do + echo " -> Waiting for port ${PORT} to free..." + sleep 0.2 + tries=$((tries+1)) + done + sleep 0.2 + echo " -> Starting Nginx" + setsid nohup $NGINX_BIN &>/dev/null & + echo " -> Starting Backend ${APP_DIR}/server..." + $APP_COMMAND + sleep 2 +} + +# configure_php: configure PHP-FPM and enable dev debug options +configure_php() { + echo "[2/3] Configuring PHP-FPM..." + sudo killall php-fpm83 &>/dev/null || true + install -d -o nginx -g www-data /run/php/ &>/dev/null + sudo sed -i "/^;pid/c\pid = /run/php/php8.3-fpm.pid" /etc/php83/php-fpm.conf + sudo sed -i 's|^listen = .*|listen = 127.0.0.1:9000|' /etc/php83/php-fpm.d/www.conf + sudo sed -i 's|fastcgi_pass .*|fastcgi_pass 127.0.0.1:9000;|' /etc/nginx/http.d/*.conf + + #increase max child process count to 10 + sudo sed -i -e 's/pm.max_children = 5/pm.max_children = 10/' /etc/php83/php-fpm.d/www.conf + + # find any line in php-fmp that starts with either ;error_log or error_log = and replace it with error_log = /app/log/app.php_errors.log + sudo sed -i '/^;*error_log\s*=/c\error_log = /app/log/app.php_errors.log' /etc/php83/php-fpm.conf + # If the line was not found, append it to the end of the file + if ! grep -q '^error_log\s*=' /etc/php83/php-fpm.conf; then + echo 'error_log = /app/log/app.php_errors.log' | sudo tee -a /etc/php83/php-fpm.conf + fi + + sudo mkdir -p /etc/php83/conf.d + sudo cp /workspaces/NetAlertX/.devcontainer/resources/99-xdebug.ini /etc/php83/conf.d/99-xdebug.ini + + sudo rm -R /var/log/php83 &>/dev/null || true + install -d -o netalertx -g www-data -m 755 var/log/php83; + + sudo chmod 644 /etc/php83/conf.d/99-xdebug.ini || true + +} + +# (duplicate start_services removed) + + + +echo "$(git rev-parse --short=8 HEAD)">/app/.VERSION +# Run the main function +main + + + diff --git a/.devcontainer/scripts/stream-logs.sh b/.devcontainer/scripts/stream-logs.sh new file mode 100755 index 00000000..f9864b29 --- /dev/null +++ b/.devcontainer/scripts/stream-logs.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Stream NetAlertX logs to stdout so the Dev Containers output channel shows them. +# This script waits briefly for the files to appear and then tails them with -F. + +LOG_FILES="/app/log/app.log /app/log/db_is_locked.log /app/log/execution_queue.log /app/log/app_front.log /app/log/app.php_errors.log /app/log/IP_changes.log /app/stderr.log /app/stdout.log" + +wait_for_files() { + # Wait up to ~10s for at least one of the files to exist + attempts=0 + while [ $attempts -lt 20 ]; do + for f in $LOG_FILES; do + if [ -f "$f" ]; then + return 0 + fi + done + attempts=$((attempts+1)) + sleep 0.5 + done + return 1 +} + +if wait_for_files; then + echo "Starting log stream for:" + for f in $LOG_FILES; do + [ -f "$f" ] && echo " $f" + done + + # Use tail -F where available. If tail -F isn't supported, tail -f is used as fallback. + # Some minimal images may have busybox tail without -F; this handles both. + if tail --version >/dev/null 2>&1; then + # GNU tail supports -F + tail -n +1 -F $LOG_FILES + else + # Fallback to -f for busybox; will exit if files rotate or do not exist initially + tail -n +1 -f $LOG_FILES + fi +else + echo "No log files appeared after wait; exiting stream script." + exit 0 +fi diff --git a/.devcontainer/xdebug-trigger.ini b/.devcontainer/xdebug-trigger.ini new file mode 100644 index 00000000..fe3c856b --- /dev/null +++ b/.devcontainer/xdebug-trigger.ini @@ -0,0 +1,11 @@ +zend_extension=xdebug.so +xdebug.mode=debug +xdebug.start_with_request=trigger +xdebug.trigger_value=VSCODE +xdebug.client_host=host.docker.internal +xdebug.client_port=9003 +xdebug.log=/var/log/xdebug.log +xdebug.log_level=7 +xdebug.idekey=VSCODE +xdebug.discover_client_host=true +xdebug.max_nesting_level=512 diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..d7f55ba5 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,48 @@ +This is NetAlertX — network monitoring & alerting. + +Purpose: Guide AI assistants to follow NetAlertX architecture, conventions, and safety practices. Be concise, opinionated, and prefer existing helpers/settings over new code or hardcoded values. + +## Architecture (what runs where) +- Backend (Python): main loop + GraphQL/REST endpoints orchestrate scans, plugins, workflows, notifications, and JSON export. + - Key: `server/__main__.py`, `server/plugin.py`, `server/initialise.py`, `server/api_server/api_server_start.py` +- Data (SQLite): persistent state in `db/app.db`; helpers in `server/database.py` and `server/db/*`. +- Frontend (Nginx + PHP + JS): UI reads JSON, triggers execution queue events. + - Key: `front/`, `front/js/common.js`, `front/php/server/*.php` +- Plugins (Python): acquisition/enrichment/publishers under `front/plugins/*` with `config.json` manifests. +- Messaging/Workflows: `server/messaging/*`, `server/workflows/*` +- API JSON Cache for UI: generated under `api/*.json` + +Backend loop phases (see `server/__main__.py` and `server/plugin.py`): `once`, `schedule`, `always_after_scan`, `before_name_updates`, `on_new_device`, `on_notification`, plus ad‑hoc `run` via execution queue. Plugins execute as scripts that write result logs for ingestion. + +## Plugin patterns that matter +- Manifest lives at `front/plugins//config.json`; `code_name` == folder, `unique_prefix` drives settings and filenames (e.g., `ARPSCAN`). +- Control via settings: `_RUN` (phase), `_RUN_SCHD` (cron-like), `_CMD` (script path), `_RUN_TIMEOUT`, `_WATCH` (diff columns). +- Data contract: scripts write `/app/log/plugins/last_result..log` (pipe‑delimited: 9 required cols + optional 4). Use `front/plugins/plugin_helper.py`’s `Plugin_Objects` to sanitize text and normalize MACs, then `write_result_file()`. +- Device import: define `database_column_definitions` when creating/updating devices; watched fields trigger notifications. + +## API/Endpoints quick map +- Flask app: `server/api_server/api_server_start.py` exposes routes like `/device/`, `/devices`, `/devices/export/{csv,json}`, `/devices/import`, `/devices/totals`, `/devices/by-status`, plus `nettools`, `events`, `sessions`, `dbquery`, `metrics`, `sync`. +- Authorization: all routes expect header `Authorization: Bearer ` via `get_setting_value('API_TOKEN')`. + +## Conventions & helpers to reuse +- Settings: add/modify via `ccd()` in `server/initialise.py` or per‑plugin manifest. Never hardcode ports or secrets; use `get_setting_value()`. +- Logging: use `logger.mylog(level, [message])`; levels: none/minimal/verbose/debug/trace. +- Time/MAC/strings: `helper.py` (`timeNowTZ`, `normalize_mac`, sanitizers). Validate MACs before DB writes. +- DB helpers: prefer `server/db/db_helper.py` functions (e.g., `get_table_json`, device condition helpers) over raw SQL in new paths. + +## Dev workflow (devcontainer) +- Services: use tasks to (re)start backend and nginx/PHP-FPM. Backend runs with debugpy on 5678; attach a Python debugger if needed. +- Run a plugin manually: `python3 front/plugins//script.py` (ensure `sys.path` includes `/app/front/plugins` and `/app/server` like the template). +- Testing: pytest available via Alpine packages. Tests live in `test/`; app code is under `server/`. PYTHONPATH is preconfigured to include workspace and `/opt/venv` site‑packages. + +## What “done right” looks like +- When adding a plugin, start from `front/plugins/__template`, implement with `plugin_helper`, define manifest settings, and wire phase via `_RUN`. Verify logs in `/app/log/plugins/` and data in `api/*.json`. +- When introducing new config, define it once (core `ccd()` or plugin manifest) and read it via helpers everywhere. +- When exposing new server functionality, add endpoints in `server/api_server/*` and keep authorization consistent; update UI by reading/writing JSON cache rather than bypassing the pipeline. + +## Useful references +- Docs: `docs/PLUGINS_DEV.md`, `docs/SETTINGS_SYSTEM.md`, `docs/API_*.md`, `docs/DEBUG_*.md` +- Logs: backend `/app/log/app.log`, plugin logs under `/app/log/plugins/`, nginx/php logs under `/var/log/*` + +Assistant expectations +- Reference concrete files/paths. Use existing helpers/settings. Keep changes idempotent and safe. Offer a quick validation step (log line, API hit, or JSON export) for anything you add. \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..15d4af64 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,34 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python Backend Debug: Attach", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + // Map workspace root to /app for PHP and other resources, plus explicit server mapping for Python. + "localRoot": "${workspaceFolder}", + "remoteRoot": "/app" + }, + { + "localRoot": "${workspaceFolder}/server", + "remoteRoot": "/app/server" + } + ] + }, + { + "name": "PHP Frontend Xdebug: Listen", + "type": "php", + "request": "launch", + "port": 9003, + "pathMappings": { + "/app": "${workspaceFolder}" + } + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..b3b546f5 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,13 @@ +{ + "terminal.integrated.suggest.enabled": true, + // Use pytest and look under the test/ folder + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": [ + "test" + ], + // Ensure VS Code uses the devcontainer virtualenv + "python.defaultInterpreterPath": "/opt/venv/bin/python", + // Let the Python extension invoke pytest via the interpreter; avoid hardcoded paths + // Removed python.testing.pytestPath and legacy pytest.command overrides +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 00000000..673a0243 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,94 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Generate Dockerfile", + "type": "shell", + "command": "${workspaceFolder:NetAlertX}/.devcontainer/scripts/generate-dockerfile.sh", + "presentation": { + "echo": true, + "reveal": "always", + "panel": "shared", + "showReuseMessage": false + }, + "problemMatcher": [], + "group": { + "kind": "build", + "isDefault": false + }, + "options": { + "cwd": "${workspaceFolder:NetAlertX}" + }, + "icon": { + "id": "tools", + "color": "terminal.ansiYellow" + } + }, + { + "label": "Re-Run Startup Script", + "type": "shell", + "command": "${workspaceFolder:NetAlertX}/.devcontainer/scripts/setup.sh", + "presentation": { + "echo": true, + "reveal": "always", + "panel": "shared", + "showReuseMessage": false + }, + "problemMatcher": [], + "icon": { + "id": "beaker", + "color": "terminal.ansiBlue" + } + }, + { + "label": "Start Backend (Python)", + "type": "shell", + "command": "/workspaces/NetAlertX/.devcontainer/scripts/restart-backend.sh", + "presentation": { + "echo": true, + "reveal": "always", + "panel": "shared", + "showReuseMessage": false, + "clear": false + }, + "problemMatcher": [], + "icon": { + "id": "debug-restart", + "color": "terminal.ansiGreen" + } + }, + { + "label": "Start Frontend (nginx and PHP-FPM)", + "type": "shell", + "command": "killall php-fpm83 nginx 2>/dev/null || true; sleep 1; php-fpm83 & nginx", + "presentation": { + "echo": true, + "reveal": "always", + "panel": "shared", + "showReuseMessage": false, + "clear": false + }, + "problemMatcher": [], + "icon": { + "id": "debug-restart", + "color": "terminal.ansiGreen" + } + }, + { + "label": "Stop Frontend & Backend Services", + "type": "shell", + "command": "pkill -f 'php-fpm83|nginx|crond|python3' || true", + "presentation": { + "echo": true, + "reveal": "always", + "panel": "shared", + "showReuseMessage": false + }, + "problemMatcher": [], + "icon": { + "id": "debug-stop", + "color": "terminal.ansiRed" + } + } + ] +} \ No newline at end of file diff --git a/front/.gitignore b/front/.gitignore new file mode 100644 index 00000000..ec7c331e --- /dev/null +++ b/front/.gitignore @@ -0,0 +1 @@ +buildtimestamp.txt \ No newline at end of file diff --git a/front/php/components/logs.php b/front/php/components/logs.php index aa8d5d52..53d9b6a1 100755 --- a/front/php/components/logs.php +++ b/front/php/components/logs.php @@ -62,7 +62,7 @@ function renderLogArea($params) { '
-
' . htmlspecialchars($fileName) . ' +
' . htmlspecialchars($filePath) . '
' . number_format((filesize($filePath) / 1000000), 2, ",", ".") . ' MB' . $downloadButtonHtml . '
diff --git a/front/php/server/db.php b/front/php/server/db.php index f0ee9f1a..0c046fcd 100755 --- a/front/php/server/db.php +++ b/front/php/server/db.php @@ -82,7 +82,7 @@ class CustomDatabaseWrapper { private $maxRetries; private $retryDelay; - public function __construct($filename, $flags = SQLITE3_OPEN_READWRITE | SQLITE3_OPEN_CREATE, $maxRetries = 3, $retryDelay = 1000, $encryptionKey = null) { + public function __construct($filename, $flags = SQLITE3_OPEN_READWRITE | SQLITE3_OPEN_CREATE, $maxRetries = 3, $retryDelay = 1000, $encryptionKey = "") { $this->sqlite = new SQLite3($filename, $flags, $encryptionKey); $this->maxRetries = $maxRetries; $this->retryDelay = $retryDelay; diff --git a/install/ubuntu24/install.ubuntu24.sh b/install/ubuntu24/install.ubuntu24.sh index 0d40672a..8164e944 100755 --- a/install/ubuntu24/install.ubuntu24.sh +++ b/install/ubuntu24/install.ubuntu24.sh @@ -14,7 +14,8 @@ echo "---------------------------------------------------------" # Set environment variables INSTALL_DIR=/app # Specify the installation directory here -INSTALLER_DIR=$INSTALL_DIR/install/ubuntu24 +INSTALL_SYSTEM_NAME=ubuntu24 +INSTALLER_DIR=$INSTALL_DIR/install/$INSTALL_SYSTEM_NAME # Check if script is run as root if [[ $EUID -ne 0 ]]; then @@ -101,5 +102,5 @@ fi # This is where we setup the virtual environment and install dependencies cd "$INSTALLER_DIR" || { echo "Failed to change directory to $INSTALLER_DIR"; exit 1; } -chmod +x "$INSTALLER_DIR/start.ubuntu24.sh" -"$INSTALLER_DIR/start.ubuntu24.sh" +chmod +x "$INSTALLER_DIR/start.$INSTALL_SYSTEM_NAME.sh" +"$INSTALLER_DIR/start.$INSTALL_SYSTEM_NAME.sh" diff --git a/install/ubuntu24/start.ubuntu24.sh b/install/ubuntu24/start.ubuntu24.sh index 0770b3b7..5564a775 100755 --- a/install/ubuntu24/start.ubuntu24.sh +++ b/install/ubuntu24/start.ubuntu24.sh @@ -10,7 +10,8 @@ echo "This script will set up and start NetAlertX on your Ubuntu24 system." INSTALL_DIR=/app # DO NOT CHANGE ANYTHING BELOW THIS LINE! -INSTALLER_DIR=$INSTALL_DIR/install/ubuntu24 +INSTALL_SYSTEM_NAME=ubuntu24 +INSTALLER_DIR=$INSTALL_DIR/install/$INSTALL_SYSTEM_NAME CONF_FILE=app.conf DB_FILE=app.db NGINX_CONF_FILE=netalertx.conf @@ -50,11 +51,12 @@ echo # Install dependencies apt-get install -y \ tini snmp ca-certificates curl libwww-perl arp-scan perl apt-utils cron \ - nginx-light php php-cgi php-fpm php-sqlite3 php-curl sqlite3 dnsutils net-tools \ + sqlite3 dnsutils net-tools mtr \ python3 python3-dev iproute2 nmap python3-pip zip usbutils traceroute nbtscan avahi-daemon avahi-utils build-essential # alternate dependencies -apt-get install nginx nginx-core mtr php-fpm php${PHPVERSION}-fpm php-cli php${PHPVERSION} php${PHPVERSION}-sqlite3 -y +# nginx-core install nginx and nginx-common as dependencies +apt-get install nginx-core php${PHPVERSION} php${PHPVERSION}-sqlite3 php php-cgi php-fpm php-sqlite3 php-curl php-fpm php${PHPVERSION}-fpm php-cli -y phpenmod -v ${PHPVERSION} sqlite3 update-alternatives --install /usr/bin/python python /usr/bin/python3 10 @@ -191,8 +193,8 @@ fi # Copy starter $DB_FILE and $CONF_FILE if they don't exist -cp --update=none "${INSTALL_PATH}/back/$CONF_FILE" "${INSTALL_PATH}/config/$CONF_FILE" -cp --update=none "${INSTALL_PATH}/back/$DB_FILE" "$FILEDB" +cp -u "${INSTALL_PATH}/back/$CONF_FILE" "${INSTALL_PATH}/config/$CONF_FILE" +cp -u "${INSTALL_PATH}/back/$DB_FILE" "$FILEDB" echo "[INSTALL] Fixing permissions after copied starter config & DB" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..015a7986 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,5 @@ +[tool.pytest.ini_options] +python_classes = ["Test", "Describe"] +python_functions = ["test_", "it_", "and_", "but_", "they_"] +python_files = ["test_*.py",] +testpaths = ["test",] \ No newline at end of file