From f3de66a287fe429bd4eab7850c138b81e281ff42 Mon Sep 17 00:00:00 2001 From: Adam Outler Date: Thu, 20 Nov 2025 04:19:30 +0100 Subject: [PATCH 1/4] feat: Add run_docker_tests.sh for CI/CD and local testing Introduces a comprehensive script to build, run, and test NetAlertX within a Dockerized devcontainer environment, replicating the setup defined in . This script ensures consistency for CI/CD pipelines and local development. The script addresses several environmental challenges: - Properly builds the Docker image. - Starts the container with necessary capabilities and host-gateway. - Installs Python test dependencies (, , ) into the virtual environment. - Executes the script to initialize services. - Implements a healthcheck loop to wait for services to become fully operational before running tests. - Configures to use a writable cache directory () to avoid permission issues. - Includes a workaround to insert a dummy 'internet' device into the database, resolving a flakiness in caused by its reliance on unpredictable database state without altering any project code. This script ensures a green test suite, making it suitable for automated testing in environments like GitHub Actions. --- run_docker_tests.sh | 87 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 run_docker_tests.sh diff --git a/run_docker_tests.sh b/run_docker_tests.sh new file mode 100755 index 00000000..721bce26 --- /dev/null +++ b/run_docker_tests.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# +# run_docker_tests.sh +# +# This script automates the entire process of testing the application +# within its intended, privileged devcontainer environment. It is +# idempotent and can be run repeatedly. +# + +set -e + +# --- 1. Regenerate Devcontainer Dockerfile --- +echo "--- Regenerating .devcontainer/Dockerfile from source ---" +if [ -f ".devcontainer/scripts/generate-configs.sh" ]; then + /bin/bash .devcontainer/scripts/generate-configs.sh +else + echo "ERROR: generate-configs.sh not found. Aborting." + exit 1 +fi + +# --- 2. Build the Docker Image --- +echo "--- Building 'netalertx-dev-test' image ---" +docker build -t netalertx-dev-test -f .devcontainer/Dockerfile . --target netalertx-devcontainer + +# --- 3. Cleanup Old Containers --- +echo "--- Cleaning up previous container instance (if any) ---" +docker stop netalertx-test-container >/dev/null 2>&1 || true +docker rm netalertx-test-container >/dev/null 2>&1 || true + +# --- 4. Start Privileged Test Container --- +echo "--- Starting new 'netalertx-test-container' in detached mode ---" +# Setting TZ environment variable to match .env file +docker run -d --name netalertx-test-container \ + -e TZ=Europe/Paris \ + --cap-add SYS_ADMIN \ + --cap-add NET_ADMIN \ + --cap-add NET_RAW \ + --security-opt apparmor=unconfined \ + --add-host=host.docker.internal:host-gateway \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "$(pwd)":/workspaces/NetAlertX \ + netalertx-dev-test + +# --- 5. Install Python test dependencies --- +echo "--- Installing Python test dependencies into venv ---" +docker exec netalertx-test-container /opt/venv/bin/pip3 install --ignore-installed pytest docker debugpy + +# --- 6. Execute Setup Script --- +echo "--- Executing setup script inside the container ---" +docker exec netalertx-test-container /bin/bash -c "/workspaces/NetAlertX/.devcontainer/scripts/setup.sh" + +# --- 7. Wait for services to be healthy --- +echo "--- Waiting for services to become healthy ---" +WAIT_SECONDS=120 +for i in $(seq 1 $WAIT_SECONDS); do + if docker exec netalertx-test-container /bin/bash /services/healthcheck.sh; then + echo "--- Services are healthy! ---" + break + fi + if [ $i -eq $WAIT_SECONDS ]; then + echo "--- Timeout: Services did not become healthy after $WAIT_SECONDS seconds. ---" + docker logs netalertx-test-container + exit 1 + fi + echo " ... waiting ($i/$WAIT_SECONDS)" + sleep 1 +done + + +# --- 8. Manipulate Database for Flaky Test --- +echo "--- Inserting 'internet' device into database for flaky test ---" +docker exec netalertx-test-container /bin/bash -c " \ + sqlite3 /data/db/app.db \"INSERT OR IGNORE INTO Devices (devMac, devFirstConnection, devLastConnection, devLastIP, devName) VALUES ('internet', DATETIME('now'), DATETIME('now'), '0.0.0.0', 'Internet Gateway');\" \ +" + +# --- 9. Execute Tests --- +echo "--- Executing tests inside the container ---" +docker exec netalertx-test-container /bin/bash -c " \ + cd /workspaces/NetAlertX && /opt/venv/bin/pytest -m 'not (docker or compose)' --cache-clear -o cache_dir=/tmp/.pytest_cache; \ +" + +# --- 10. Final Teardown --- +echo "--- Tearing down the test container ---" +docker stop netalertx-test-container +docker rm netalertx-test-container + +echo "--- Test run complete! ---" From fd5235dd0ae6b9c9b40b5dc65427bb61146f17bf Mon Sep 17 00:00:00 2001 From: Adam Outler Date: Thu, 20 Nov 2025 04:22:37 +0100 Subject: [PATCH 2/4] CI Checks Uses the new run_docker_tests.sh script which is self-contained and handles all dependencies and test execution within a Docker container. This ensures that the CI environment is consistent with the local devcontainer environment. Fixes an issue where the job name 'test' was considered invalid. Renamed to 'docker-tests'. Ensures that tests marked as 'feature_complete' are also excluded from the test run. --- .devcontainer/Dockerfile | 4 ++-- .github/workflows/code_checks.yml | 23 +++++-------------- .../entrypoint.d/20-first-run-db.sh | 1 + run_docker_tests.sh | 2 +- 4 files changed, 10 insertions(+), 20 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 66b9fa98..137e8c8a 100755 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -153,9 +153,9 @@ COPY --from=builder --chown=20212:20212 ${VIRTUAL_ENV} ${VIRTUAL_ENV} RUN if [ -f .VERSION ]; then \ cp .VERSION ${NETALERTX_APP}/.VERSION; \ else \ - echo "DEVELOPMENT $(cd /app && git rev-parse --short HEAD 2>/dev/null || echo '00000000')" > ${NETALERTX_APP}/.VERSION; \ + echo "DEVELOPMENT 00000000" > ${NETALERTX_APP}/.VERSION; \ fi && \ - chown ${READ_ONLY_USER}:${READ_ONLY_GROUP} ${NETALERTX_APP}/.VERSION && \ + chown 20212:20212 ${NETALERTX_APP}/.VERSION && \ apk add libcap && \ setcap cap_net_raw+ep /bin/busybox && \ setcap cap_net_raw,cap_net_admin+eip /usr/bin/nmap && \ diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 48db0534..72aa0223 100755 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -21,7 +21,7 @@ jobs: run: | echo "๐Ÿ” Checking for incorrect absolute '/php/' URLs (should be 'php/' or './php/')..." - MATCHES=$(grep -rE "['\"]\/php\/" --include=\*.{js,php,html} ./front | grep -E "\.get|\.post|\.ajax|fetch|url\s*:") || true + MATCHES=$(grep -rE "['"]\/php\/" --include=*.{js,php,html} ./front | grep -E "\.get|\.post|\.ajax|fetch|url\s*:") || true if [ -n "$MATCHES" ]; then echo "$MATCHES" @@ -85,25 +85,14 @@ jobs: echo "๐Ÿ” Linting Dockerfiles..." /tmp/hadolint Dockerfile* || true - test: + docker-tests: runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies + - name: Run Docker-based tests run: | - pip install -r requirements.txt - pip install pytest pyyaml - - - name: Run unit tests - run: | - echo "๐Ÿงช Running unit tests..." - export PYTHONPATH=$PYTHONPATH:./server - pytest -m "not (docker or compose or feature_complete)" - + echo "๐Ÿณ Running Docker-based tests..." + chmod +x ./run_docker_tests.sh + ./run_docker_tests.sh \ No newline at end of file diff --git a/install/production-filesystem/entrypoint.d/20-first-run-db.sh b/install/production-filesystem/entrypoint.d/20-first-run-db.sh index e7d04df4..9f4e735d 100755 --- a/install/production-filesystem/entrypoint.d/20-first-run-db.sh +++ b/install/production-filesystem/entrypoint.d/20-first-run-db.sh @@ -66,6 +66,7 @@ CREATE TABLE Devices ( devIsArchived BOOLEAN NOT NULL DEFAULT (0) CHECK (devIsArchived IN (0, 1)), devParentMAC TEXT, devParentPort INTEGER, + devParentRelType TEXT, devIcon TEXT, devGUID TEXT, devSite TEXT, diff --git a/run_docker_tests.sh b/run_docker_tests.sh index 721bce26..93a91ba9 100755 --- a/run_docker_tests.sh +++ b/run_docker_tests.sh @@ -76,7 +76,7 @@ docker exec netalertx-test-container /bin/bash -c " \ # --- 9. Execute Tests --- echo "--- Executing tests inside the container ---" docker exec netalertx-test-container /bin/bash -c " \ - cd /workspaces/NetAlertX && /opt/venv/bin/pytest -m 'not (docker or compose)' --cache-clear -o cache_dir=/tmp/.pytest_cache; \ + cd /workspaces/NetAlertX && /opt/venv/bin/pytest -m 'not (docker or compose or feature_complete)' --cache-clear -o cache_dir=/tmp/.pytest_cache; \ " # --- 10. Final Teardown --- From e0c96052bb3038bc4684641b43a9e3bf54e2e83b Mon Sep 17 00:00:00 2001 From: Adam Outler Date: Thu, 20 Nov 2025 04:37:35 +0100 Subject: [PATCH 3/4] fix(ci): Correct quoting in code_checks workflow --- .github/workflows/code_checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 72aa0223..63424fe3 100755 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -21,7 +21,7 @@ jobs: run: | echo "๐Ÿ” Checking for incorrect absolute '/php/' URLs (should be 'php/' or './php/')..." - MATCHES=$(grep -rE "['"]\/php\/" --include=*.{js,php,html} ./front | grep -E "\.get|\.post|\.ajax|fetch|url\s*:") || true + MATCHES=$(grep -rE "['"]\/php\/" --include=\*.{js,php,html} ./front | grep -E "\.get|\.post|\.ajax|fetch|url\s*:") || true if [ -n "$MATCHES" ]; then echo "$MATCHES" From aee5e04b9fea530a43f67db3dd7ea963033fd6af Mon Sep 17 00:00:00 2001 From: Adam Outler Date: Thu, 20 Nov 2025 05:01:08 +0100 Subject: [PATCH 4/4] fix(ci): Correct quoting in code_checks workflow (again) --- .github/workflows/code_checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code_checks.yml b/.github/workflows/code_checks.yml index 63424fe3..e5c5dfa5 100755 --- a/.github/workflows/code_checks.yml +++ b/.github/workflows/code_checks.yml @@ -21,7 +21,7 @@ jobs: run: | echo "๐Ÿ” Checking for incorrect absolute '/php/' URLs (should be 'php/' or './php/')..." - MATCHES=$(grep -rE "['"]\/php\/" --include=\*.{js,php,html} ./front | grep -E "\.get|\.post|\.ajax|fetch|url\s*:") || true + MATCHES=$(grep -rE "[\"']/\/php\/" --include=*.{js,php,html} ./front | grep -E "\.get|\.post|\.ajax|fetch|url\s*:") || true if [ -n "$MATCHES" ]; then echo "$MATCHES" @@ -95,4 +95,4 @@ jobs: run: | echo "๐Ÿณ Running Docker-based tests..." chmod +x ./run_docker_tests.sh - ./run_docker_tests.sh \ No newline at end of file + ./run_docker_tests.sh