Merge pull request #1302 from adamoutler/supercronic

Replace crond with Supercronic, improve cron logging & backend restart behavior

.devcontainer/Dockerfile

@@ -64,6 +64,7 @@ ENV LOG_IP_CHANGES=${NETALERTX_LOG}/IP_changes.log
 ENV LOG_APP=${NETALERTX_LOG}/app.log
 ENV LOG_APP_FRONT=${NETALERTX_LOG}/app_front.log
 ENV LOG_REPORT_OUTPUT_TXT=${NETALERTX_LOG}/report_output.txt
+ENV LOG_CRON=${NETALERTX_LOG}/supercronic
 ENV LOG_DB_IS_LOCKED=${NETALERTX_LOG}/db_is_locked.log
 ENV LOG_REPORT_OUTPUT_HTML=${NETALERTX_LOG}/report_output.html
 ENV LOG_STDERR=${NETALERTX_LOG}/stderr.log
@@ -71,7 +72,7 @@ ENV LOG_APP_PHP_ERRORS=${NETALERTX_LOG}/app.php_errors.log
 ENV LOG_EXECUTION_QUEUE=${NETALERTX_LOG}/execution_queue.log
 ENV LOG_REPORT_OUTPUT_JSON=${NETALERTX_LOG}/report_output.json
 ENV LOG_STDOUT=${NETALERTX_LOG}/stdout.log
-ENV LOG_CROND=${NETALERTX_LOG}/crond.log
+ENV LOG_CRON=${NETALERTX_LOG}/cron.log
 ENV LOG_NGINX_ERROR=${NETALERTX_LOG}/nginx-error.log
 
 # System Services configuration files
@@ -81,11 +82,11 @@ ENV SYSTEM_SERVICES_SCRIPTS=${SYSTEM_SERVICES}/scripts
 ENV SYSTEM_SERVICES_CONFIG=${SYSTEM_SERVICES}/config
 ENV SYSTEM_NGINX_CONFIG=${SYSTEM_SERVICES_CONFIG}/nginx
 ENV SYSTEM_NGINX_CONFIG_TEMPLATE=${SYSTEM_NGINX_CONFIG}/netalertx.conf.template
+ENV SYSTEM_SERVICES_CONFIG_CRON=${SYSTEM_SERVICES_CONFIG}/cron
 ENV SYSTEM_SERVICES_ACTIVE_CONFIG=/tmp/nginx/active-config
 ENV SYSTEM_SERVICES_ACTIVE_CONFIG_FILE=${SYSTEM_SERVICES_ACTIVE_CONFIG}/nginx.conf
 ENV SYSTEM_SERVICES_PHP_FOLDER=${SYSTEM_SERVICES_CONFIG}/php
 ENV SYSTEM_SERVICES_PHP_FPM_D=${SYSTEM_SERVICES_PHP_FOLDER}/php-fpm.d
-ENV SYSTEM_SERVICES_CROND=${SYSTEM_SERVICES_CONFIG}/crond
 ENV SYSTEM_SERVICES_RUN=/tmp/run
 ENV SYSTEM_SERVICES_RUN_TMP=${SYSTEM_SERVICES_RUN}/tmp
 ENV SYSTEM_SERVICES_RUN_LOG=${SYSTEM_SERVICES_RUN}/logs
@@ -119,7 +120,7 @@ ENV LANG=C.UTF-8
 RUN apk add --no-cache bash mtr libbsd zip lsblk tzdata curl arp-scan iproute2 iproute2-ss nmap \
     nmap-scripts traceroute nbtscan net-tools net-snmp-tools bind-tools awake ca-certificates \
     sqlite php83 php83-fpm php83-cgi php83-curl php83-sqlite3 php83-session python3 envsubst \
-    nginx shadow && \
+    nginx supercronic shadow && \
     rm -Rf /var/cache/apk/*  && \
     rm -Rf /etc/nginx && \
     addgroup -g 20211 ${NETALERTX_GROUP} && \
@@ -165,7 +166,7 @@ RUN if [ -f .VERSION ]; then \
     setcap cap_net_raw,cap_net_admin+eip $(readlink -f ${VIRTUAL_ENV_BIN}/python) && \
     /bin/sh /build/init-nginx.sh && \
     /bin/sh /build/init-php-fpm.sh && \
-    /bin/sh /build/init-crond.sh && \
+    /bin/sh /build/init-cron.sh && \
     /bin/sh /build/init-backend.sh && \
     rm -rf /build && \
     apk del libcap && \

.devcontainer/scripts/setup.sh

@@ -26,7 +26,7 @@ LOG_FILES=(
   LOG_EXECUTION_QUEUE
   LOG_APP_PHP_ERRORS
   LOG_IP_CHANGES
-  LOG_CROND
+  LOG_CRON
   LOG_REPORT_OUTPUT_TXT
   LOG_REPORT_OUTPUT_HTML
   LOG_REPORT_OUTPUT_JSON

.github/copilot-instructions.md

@@ -83,3 +83,9 @@ Backend loop phases (see `server/__main__.py` and `server/plugin.py`): `once`, `
 - Be sure to offer choices when appropriate.
 - Always understand the intent of the user's request and undo/redo as needed.
 - Above all, use the simplest possible code that meets the need so it can be easily audited and maintained.
+- Always leave logging enabled. If there is a possiblity it will be difficult to debug with current logging, add more logging.
+- Always run the testFailure tool before executing any tests to gather current failure information and avoid redundant runs.
+- Always prioritize using the appropriate tools in the environment first. As an example if a test is failing use `testFailure` then `runTests`. Never `runTests` first. 
+- Docker tests take an extremely long time to run.  Avoid changes to docker or tests until you've examined the exisiting testFailures and runTests results.
+- Environment tools are designed specifically for your use in this project and running them in this order will give you the best results. 
+

.github/workflows/code_checks.yml

@@ -95,5 +95,5 @@ jobs:
       - name: Run Docker-based tests
         run: |
           echo "🐳 Running Docker-based tests..."
-          chmod +x ./run_docker_tests.sh
-          ./run_docker_tests.sh
+          chmod +x ./test/docker_tests/run_docker_tests.sh
+          ./test/docker_tests/run_docker_tests.sh

Dockerfile

@@ -68,7 +68,7 @@ ENV LOG_APP_PHP_ERRORS=${NETALERTX_LOG}/app.php_errors.log
 ENV LOG_EXECUTION_QUEUE=${NETALERTX_LOG}/execution_queue.log
 ENV LOG_REPORT_OUTPUT_JSON=${NETALERTX_LOG}/report_output.json
 ENV LOG_STDOUT=${NETALERTX_LOG}/stdout.log
-ENV LOG_CROND=${NETALERTX_LOG}/crond.log
+ENV LOG_CRON=${NETALERTX_LOG}/cron.log
 ENV LOG_NGINX_ERROR=${NETALERTX_LOG}/nginx-error.log
 
 # System Services configuration files
@@ -78,11 +78,11 @@ ENV SYSTEM_SERVICES_SCRIPTS=${SYSTEM_SERVICES}/scripts
 ENV SYSTEM_SERVICES_CONFIG=${SYSTEM_SERVICES}/config
 ENV SYSTEM_NGINX_CONFIG=${SYSTEM_SERVICES_CONFIG}/nginx
 ENV SYSTEM_NGINX_CONFIG_TEMPLATE=${SYSTEM_NGINX_CONFIG}/netalertx.conf.template
+ENV SYSTEM_SERVICES_CONFIG_CRON=${SYSTEM_SERVICES_CONFIG}/cron
 ENV SYSTEM_SERVICES_ACTIVE_CONFIG=/tmp/nginx/active-config
 ENV SYSTEM_SERVICES_ACTIVE_CONFIG_FILE=${SYSTEM_SERVICES_ACTIVE_CONFIG}/nginx.conf
 ENV SYSTEM_SERVICES_PHP_FOLDER=${SYSTEM_SERVICES_CONFIG}/php
 ENV SYSTEM_SERVICES_PHP_FPM_D=${SYSTEM_SERVICES_PHP_FOLDER}/php-fpm.d
-ENV SYSTEM_SERVICES_CROND=${SYSTEM_SERVICES_CONFIG}/crond
 ENV SYSTEM_SERVICES_RUN=/tmp/run
 ENV SYSTEM_SERVICES_RUN_TMP=${SYSTEM_SERVICES_RUN}/tmp
 ENV SYSTEM_SERVICES_RUN_LOG=${SYSTEM_SERVICES_RUN}/logs
@@ -116,7 +116,7 @@ ENV LANG=C.UTF-8
 RUN apk add --no-cache bash mtr libbsd zip lsblk tzdata curl arp-scan iproute2 iproute2-ss nmap \
     nmap-scripts traceroute nbtscan net-tools net-snmp-tools bind-tools awake ca-certificates \
     sqlite php83 php83-fpm php83-cgi php83-curl php83-sqlite3 php83-session python3 envsubst \
-    nginx shadow && \
+    nginx supercronic shadow && \
     rm -Rf /var/cache/apk/*  && \
     rm -Rf /etc/nginx && \
     addgroup -g 20211 ${NETALERTX_GROUP} && \
@@ -162,7 +162,7 @@ RUN if [ -f .VERSION ]; then \
     setcap cap_net_raw,cap_net_admin+eip $(readlink -f ${VIRTUAL_ENV_BIN}/python) && \
     /bin/sh /build/init-nginx.sh && \
     /bin/sh /build/init-php-fpm.sh && \
-    /bin/sh /build/init-crond.sh && \
+    /bin/sh /build/init-cron.sh && \
     /bin/sh /build/init-backend.sh && \
     rm -rf /build && \
     apk del libcap && \

Dockerfile.debian

@@ -72,7 +72,7 @@ ENV LOG_APP_PHP_ERRORS=${NETALERTX_LOG}/app.php_errors.log
 ENV LOG_EXECUTION_QUEUE=${NETALERTX_LOG}/execution_queue.log
 ENV LOG_REPORT_OUTPUT_JSON=${NETALERTX_LOG}/report_output.json
 ENV LOG_STDOUT=${NETALERTX_LOG}/stdout.log
-ENV LOG_CROND=${NETALERTX_LOG}/crond.log
+ENV LOG_CRON=${NETALERTX_LOG}/cron.log
 ENV LOG_NGINX_ERROR=${NETALERTX_LOG}/nginx-error.log
 
 # System Services configuration files

back/cron_script.sh

@@ -3,12 +3,17 @@ export INSTALL_DIR=/app
 
 LOG_FILE="${INSTALL_DIR}/log/execution_queue.log"
 
-# Check if there are any entries with cron_restart_backend
-if grep -q "cron_restart_backend" "$LOG_FILE"; then
-  # Restart python application using s6
-  s6-svc -r /var/run/s6-rc/servicedirs/netalertx
-  echo 'done'
+if [ -f "${LOG_EXECUTION_QUEUE}" ] && grep -q "cron_restart_backend" "${LOG_EXECUTION_QUEUE}"; then
+  echo "$(date): Restarting backend triggered by cron_restart_backend"
+  killall python3 || echo "killall python3 failed or no process found"
+  sleep 2
+  /services/start-backend.sh &
 
   # Remove all lines containing cron_restart_backend from the log file
-  sed -i '/cron_restart_backend/d' "$LOG_FILE"
+  # Atomic replacement with temp file. grep returns 1 if no lines selected (file becomes empty), which is valid here.
+  grep -v "cron_restart_backend" "${LOG_EXECUTION_QUEUE}" > "${LOG_EXECUTION_QUEUE}.tmp"
+  RC=$?
+  if [ $RC -eq 0 ] || [ $RC -eq 1 ]; then
+    mv "${LOG_EXECUTION_QUEUE}.tmp" "${LOG_EXECUTION_QUEUE}"
+  fi
 fi

front/php/components/logs_defaults.json

@@ -107,11 +107,11 @@
         "buttons": [
             {
                 "labelStringCode": "Maint_PurgeLog",
-                "event": "logManage('crond.log', 'cleanLog')"
+                "event": "logManage('cron.log', 'cleanLog')"
             }
         ],
-    "fileName": "crond.log",
-    "filePath": "__NETALERTX_LOG__/crond.log",
+    "fileName": "cron.log",
+    "filePath": "__NETALERTX_LOG__/cron.log",
         "textAreaCssClass": "logs logs-small"
     }
 ]

front/php/server/util.php

@@ -274,7 +274,7 @@ function cleanLog($logFile)
 
   $path = "";
 
-  $allowedFiles = ['app.log', 'app_front.log', 'IP_changes.log', 'stdout.log', 'stderr.log', 'app.php_errors.log', 'execution_queue.log', 'db_is_locked.log', 'nginx-error.log', 'crond.log'];
+  $allowedFiles = ['app.log', 'app_front.log', 'IP_changes.log', 'stdout.log', 'stderr.log', 'app.php_errors.log', 'execution_queue.log', 'db_is_locked.log', 'nginx-error.log', 'cron.log'];
   
   if(in_array($logFile, $allowedFiles))
   {

install/production-filesystem/build/init-cron.sh

@@ -0,0 +1,5 @@
+
+#!/bin/bash
+echo "Initializing cron..."
+ # Placeholder for cron initialization commands
+echo "cron initialized."

install/production-filesystem/build/init-crond.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-echo "Initializing crond..."
-#Future crond initializations can go here.
-echo "crond initialized."

install/production-filesystem/entrypoint.sh

@@ -274,7 +274,7 @@ trap on_signal INT TERM
 # Only start crond scheduler on Alpine (non-Debian) environments
 # Debian typically uses systemd or other schedulers
 if [ "${ENVIRONMENT:-}" ] && [ "${ENVIRONMENT:-}" != "debian" ]; then
-    add_service "/services/start-crond.sh" "crond"
+    add_service "/services/start-cron.sh" "supercronic"
 fi
 
 # Start core frontend and backend services
@@ -290,8 +290,6 @@ add_service "${SYSTEM_SERVICES}/start-backend.sh" "python3"
 # Useful for devcontainer debugging where individual services need to be debugged
 if [ "${NETALERTX_DEBUG:-0}" -eq 1 ]; then
 	echo "NETALERTX_DEBUG is set to 1, will not shut down other services if one fails."
-	wait
-	exit $?
 fi
 
 ################################################################################
@@ -316,10 +314,25 @@ while [ -n "${SERVICES}" ]; do
         if ! is_pid_active "${pid}"; then
             wait "${pid}" 2>/dev/null
             status=$?
+
+            # Handle intentional backend restart
+            if [ "${name}" = "python3" ] && [ -f "/tmp/backend_restart_pending" ]; then
+                echo "🔄 Backend restart requested via marker file."
+                rm -f "/tmp/backend_restart_pending"
+                remove_service "${pid}"
+                add_service "${SYSTEM_SERVICES}/start-backend.sh" "python3"
+                continue
+            fi
+
             FAILED_STATUS=$status
             FAILED_NAME="${name}"
             remove_service "${pid}"
-            handle_exit
+            
+            if [ "${NETALERTX_DEBUG:-0}" -eq 1 ]; then
+                echo "⚠️ Service ${name} exited with status ${status}. Debug mode active - continuing."
+            else
+                handle_exit
+            fi
         fi
 
     done

install/production-filesystem/services/config/cron/crontab

@@ -1,4 +1,4 @@
 # Every minute check for cron jobs
 * * * * * /services/scripts/cron_script.sh
 # Update vendors 4x/d
-0 */6 * * * /services/scripts/update_vendors.sh
+0 */6 * * * /services/scripts/update_vendors.sh

install/production-filesystem/services/healthcheck.sh

@@ -21,10 +21,10 @@ log_success() {
 }
 
 # 1. Check if crond is running
-if pgrep -f "crond" > /dev/null; then
-    log_success "crond is running"
+if pgrep -f "supercronic" > /dev/null; then
+    log_success "supercronic is running"
 else
-    log_error "crond is not running"
+    log_error "supercronic is not running"
 fi
 
 # 2. Check if php-fpm is running

install/production-filesystem/services/scripts/cron_script.sh

@@ -5,12 +5,15 @@ export INSTALL_DIR=/app
 
 # Check if there are any entries with cron_restart_backend
 if grep -q "cron_restart_backend" "${LOG_EXECUTION_QUEUE}"; then
-  killall python3
-  sleep 2
-  /services/start-backend.sh &
+  echo "$(date): Restarting backend triggered by cron_restart_backend"
+  
+  # Create marker for entrypoint.sh to restart the service instead of killing the container
+  touch /tmp/backend_restart_pending
+  
+  killall python3 || echo "killall python3 failed or no process found"
 
   # Remove all lines containing cron_restart_backend from the log file
   # Atomic replacement with temp file
-  grep -v "cron_restart_backend" "${LOG_EXECUTION_QUEUE}" > "${LOG_EXECUTION_QUEUE}.tmp" && \
-    mv "${LOG_EXECUTION_QUEUE}.tmp" "${LOG_EXECUTION_QUEUE}"
+  grep -v "cron_restart_backend" "${LOG_EXECUTION_QUEUE}" > "${LOG_EXECUTION_QUEUE}.tmp"
+  mv "${LOG_EXECUTION_QUEUE}.tmp" "${LOG_EXECUTION_QUEUE}"
 fi

install/production-filesystem/services/start-cron.sh

@@ -6,7 +6,7 @@ crond_pid=""
 
 cleanup() {
 	status=$?
-	echo "Crond stopped! (exit ${status})"
+	echo "Supercronic stopped! (exit ${status})"
 }
 
 forward_signal() {
@@ -23,11 +23,16 @@ done
 trap cleanup EXIT
 trap forward_signal INT TERM
 
-echo "Starting /usr/sbin/crond -c \"${SYSTEM_SERVICES_CROND}\" -f -L \"${LOG_CROND}\" >>\"${LOG_CROND}\" 2>&1 &"
+CRON_OPTS="--quiet"
+if [ "${NETALERTX_DEBUG:-0}" -eq 1 ]; then
+  CRON_OPTS="--debug"
+fi
 
-/usr/sbin/crond -c "${SYSTEM_SERVICES_CROND}" -f -L "${LOG_CROND}" >>"${LOG_CROND}" 2>&1 &
+echo "Starting supercronic ${CRON_OPTS} \"${SYSTEM_SERVICES_CONFIG_CRON}/crontab\" >>\"${LOG_CRON}\" 2>&1 &"
+
+supercronic ${CRON_OPTS} "${SYSTEM_SERVICES_CONFIG_CRON}/crontab" >>"${LOG_CRON}" 2>&1 &
 crond_pid=$!
 
 wait "${crond_pid}"; status=$?
 echo -ne " done"
-exit ${status}
+exit ${status}

test/docker_tests/test_mount_diagnostics_pytest.py

@@ -271,7 +271,7 @@ def create_test_scenarios() -> List[TestScenario]:
             compose_file = f"docker-compose.mount-test.{path_name}_{scenario_name}.yml"
 
             # Determine expected exit code
-            expected_exit_code = 1 if scenario_name == "unwritable" else 0
+            expected_exit_code = 1 if expected_issues and not (path_name == "active_config" and scenario_name == "unwritable") else 0
 
             scenarios.append(
                 TestScenario(