DBCLNP chore

back/app.conf

@@ -19,7 +19,7 @@
 
 SCAN_SUBNETS=['192.168.1.0/24 --interface=eth0']
 TIMEZONE='Europe/Berlin'
-LOADED_PLUGINS = ['ARPSCAN','CSVBCKP','DBCLNP', 'INTRNT','MAINT','NEWDEV','NSLOOKUP','NTFPRCS', 'AVAHISCAN', 'PHOLUS','SETPWD','SMTP', 'SYNC', 'VNDRPDT', 'WORKFLOWS']
+LOADED_PLUGINS = ['ARPSCAN','CSVBCKP','DBCLNP', 'INTRNT','MAINT','NEWDEV','NSLOOKUP','NTFPRCS', 'AVAHISCAN', 'SETPWD','SMTP', 'SYNC', 'VNDRPDT', 'WORKFLOWS']
 
 DAYS_TO_KEEP_EVENTS=90
 # Used for generating links in emails. Make sure not to add a trailing slash!
@@ -28,7 +28,6 @@ REPORT_DASHBOARD_URL='http://netalertx'
 # Make sure at least these scanners are enabled for new installs, other defaults are taken from the config.json
 INTRNT_RUN='schedule'
 ARPSCAN_RUN='schedule'
-PHOLUS_RUN='on_new_device'
 NSLOOKUP_RUN='before_name_updates'
 
 # Email 

docs/PERFORMANCE.md

@@ -0,0 +1,24 @@
+# Performance tips
+
+The application runs regular maintenance a DB cleanup tasks. If these tasks fail, you might encounter performance issues. 
+
+Most performance issues are caused by a big database or large log files. Enabling unnecessary plugins will also lead to performance degradation. 
+
+You can always check the size of your database and database tables under the Maintenance page. 
+
+![Db size check](/docs/img/PERFORMANCE/db_size_check.png)
+
+> [!NOTE]
+> For around 100 devices the database should be approximately `50MB` and none of the entries (rows) should exceed the value of `10 000` on a healthy system. 
+
+## Maintenance plugins
+
+There are 2 plugins responsible for maintaining the overal health of the application. One is responsible for the database cleanup and one for other tasks, such as log cleanup. 
+
+### DB Cleanup (DBCLNP)
+
+The database cleanup plugin. Check details and related setting in the [DB Cleanup plugin docs](/front/plugins/db_cleanup/README.md). Make sure the plugin is not failing by checking the logs. Try changing the schedule `DBCLNP_RUN_SCHD` and the timeout `DBCLNP_RUN_TIMEOUT` (increase) if the plugin is failing to execute.
+
+### Maintenance (MAINT)
+
+The maintenance plugin. Check details and related setting in the [Maintenance plugin docs](/front/plugins/maintenance/README.md). Make sure the plugin is not failing by checking the logs. Try changing the schedule `MAINT_RUN_SCHD` and the timeout `MAINT_RUN_TIMEOUT` (increase) if the plugin is failing to execute.

front/plugins/avahi_scan/avahi_scan.py

@@ -41,9 +41,6 @@ def main():
     # timeout = get_setting_value('AVAHI_RUN_TIMEOUT')
     timeout = 20
     
-    # ensure service is running
-    ensure_avahi_running()
-
     # Create a database connection
     db = DB()  # instance of class DB
     db.open()
@@ -57,7 +54,7 @@ def main():
     # Retrieve devices
     unknown_devices = device_handler.getUnknown()
     
-    # # Mock list of devices (replace with actual device_handler.getUnknown() in production)
+    # Mock list of devices (replace with actual device_handler.getUnknown() in production)
     # unknown_devices = [
     #     {'dev_MAC': '00:11:22:33:44:55', 'dev_LastIP': '192.168.1.121'},
     #     {'dev_MAC': '00:11:22:33:44:56', 'dev_LastIP': '192.168.1.9'},
@@ -65,11 +62,16 @@ def main():
     # ]
 
     mylog('verbose', [f'[{pluginName}] Unknown devices count: {len(unknown_devices)}'])   
+    
+    if len(unknown_devices) > 0:
+        # ensure service is running
+        ensure_avahi_running()
 
     for device in unknown_devices:
         domain_name = execute_name_lookup(device['dev_LastIP'], timeout)
 
-        if domain_name != '':
+        #  check if found and not a timeout ('to')
+        if domain_name != '' and domain_name != 'to': 
             plugin_objects.add_object(
             # "MAC", "IP", "Server", "Name"
             primaryId   = device['dev_MAC'],
@@ -140,18 +142,28 @@ def execute_name_lookup(ip, timeout):
     return ''   
 
 # Function to ensure Avahi and its dependencies are running
-def ensure_avahi_running():
+def ensure_avahi_running(attempt=1, max_retries=2):
     """
-    Ensure that D-Bus is running and the Avahi daemon is started.
+    Ensure that D-Bus is running and the Avahi daemon is started, with recursive retry logic.
     """
-    mylog('verbose', [f'[{pluginName}] Ensuring D-Bus and Avahi daemon are running...'])
-    
-    # # Install D-Bus if not already installed
-    # try:
-    #     subprocess.run(['apk', 'add', 'dbus'], check=True)
-    # except subprocess.CalledProcessError as e:
-    #     mylog('verbose', [f'[{pluginName}] ⚠ ERROR - Failed to install D-Bus: {e.output}'])
-    #     return
+    mylog('verbose', [f'[{pluginName}] Attempt {attempt} - Ensuring D-Bus and Avahi daemon are running...'])
+
+    # Check rc-status
+    try:
+        subprocess.run(['rc-status'], check=True)
+    except subprocess.CalledProcessError as e:
+        mylog('verbose', [f'[{pluginName}] ⚠ ERROR - Failed to check rc-status: {e.output}'])
+        return
+
+    # Create OpenRC soft level
+    subprocess.run(['touch', '/run/openrc/softlevel'], check=True)
+
+    # Add Avahi daemon to runlevel
+    try:
+        subprocess.run(['rc-update', 'add', 'avahi-daemon'], check=True)
+    except subprocess.CalledProcessError as e:
+        mylog('verbose', [f'[{pluginName}] ⚠ ERROR - Failed to add Avahi to runlevel: {e.output}'])
+        return
 
     # Start the D-Bus service
     try:
@@ -160,26 +172,36 @@ def ensure_avahi_running():
         mylog('verbose', [f'[{pluginName}] ⚠ ERROR - Failed to start D-Bus: {e.output}'])
         return
 
-    # Create OpenRC soft level if needed
-    subprocess.run(['touch', '/run/openrc/softlevel'], check=True)
-    
-    # Add Avahi daemon to runlevel
-    try:
-        subprocess.run(['rc-update', 'add', 'avahi-daemon'], check=True)
-    except subprocess.CalledProcessError as e:
-        mylog('verbose', [f'[{pluginName}] ⚠ ERROR - Failed to add Avahi to runlevel: {e.output}'])
+    # Check Avahi status
+    status_output = subprocess.run(['rc-service', 'avahi-daemon', 'status'], capture_output=True, text=True)
+    if 'started' in status_output.stdout:
+        mylog('verbose', [f'[{pluginName}] Avahi Daemon is already running.'])
         return
-    
+
+    mylog('verbose', [f'[{pluginName}] Avahi Daemon is not running, attempting to start... (Attempt {attempt})'])
+
     # Start the Avahi daemon
     try:
         subprocess.run(['rc-service', 'avahi-daemon', 'start'], check=True)
     except subprocess.CalledProcessError as e:
         mylog('verbose', [f'[{pluginName}] ⚠ ERROR - Failed to start Avahi daemon: {e.output}'])
-        return
-    
-    # Check status
+
+    # Check status after starting
     status_output = subprocess.run(['rc-service', 'avahi-daemon', 'status'], capture_output=True, text=True)
-    mylog('verbose', [f'[{pluginName}] Avahi Daemon Status: {status_output.stdout.strip()}'])
+    if 'started' in status_output.stdout:
+        mylog('verbose', [f'[{pluginName}] Avahi Daemon successfully started.'])
+        return
+
+    # Retry if not started and attempts are left
+    if attempt < max_retries:
+        mylog('verbose', [f'[{pluginName}] Retrying... ({attempt + 1}/{max_retries})'])
+        ensure_avahi_running(attempt + 1, max_retries)
+    else:
+        mylog('verbose', [f'[{pluginName}] ⚠ ERROR - Avahi Daemon failed to start after {max_retries} attempts.'])
+
+    # rc-update add avahi-daemon
+    # rc-service avahi-daemon status
+    # rc-service avahi-daemon start
 
 if __name__ == '__main__':
     main()

front/plugins/db_cleanup/README.md

@@ -2,6 +2,23 @@
 
 Plugin to run regular database cleanup tasks. It is strongly recommended to have an hourly or at least daily schedule running.
 
+The database cleanup plugin (DBCLNP) helps maintain the system by removing outdated or unnecessary data, ensuring smooth operation. Below are the key settings that control the cleanup process:
+
+- **`PLUGINS_KEEP_HIST`**:  
+  Specifies how many historical records to retain for each plugin. Recommended value: `500` entries.
+
+- **`HRS_TO_KEEP_NEWDEV`**:  
+  Defines how long, in hours, newly discovered device records should be kept. Once the specified time has passed, the records will be deleted if tehy still are marked as NEW. Recommended value: `0` (no auto delete).
+
+- **`DAYS_TO_KEEP_EVENTS`**:  
+  Specifies the number of days to retain event logs. Event entries older than the given number of days will be automatically deleted during cleanup. Recommended value: `30` days.
+
+- **`PHOLUS_DAYS_DATA`**:  
+  Deletes all data older than specified number of days for teh Pholus plugin used for name discovery. Recommended value: `30` days.
+
+By fine-tuning these settings, you ensure that the database remains optimized, preventing performance degradation in the NetAlertX system.
+
+
 ### Usage
 
 - Check the Settings page for details.

front/plugins/db_cleanup/config.json

@@ -25,24 +25,7 @@
       "string": "A plugin to schedule database cleanup & upkeep tasks."
     }
   ],
-  "params": [
-    {
-      "name": "pluginskeephistory",
-      "type": "setting",
-      "value": "PLUGINS_KEEP_HIST"
-    },
-    {
-      "name": "daystokeepevents",
-      "type": "setting",
-      "value": "DAYS_TO_KEEP_EVENTS"
-    },
-    {
-      "name": "hourstokeepnewdevice",
-      "type": "setting",
-      "value": "HRS_TO_KEEP_NEWDEV"
-    }
-  ],
-
+  "params": [],
   "settings": [
     {
       "function": "RUN",
@@ -89,7 +72,7 @@
           }
         ]
       },
-      "default_value": "python3 /app/front/plugins/db_cleanup/script.py pluginskeephistory={pluginskeephistory} hourstokeepnewdevice={hourstokeepnewdevice} daystokeepevents={daystokeepevents} pholuskeepdays={pholuskeepdays}",
+      "default_value": "python3 /app/front/plugins/db_cleanup/script.py",
       "options": [],
       "localized": ["name", "description"],
       "name": [
@@ -150,14 +133,6 @@
         {
           "language_code": "en_us",
           "string": "Only enabled if you select <code>schedule</code> in the <a href=\"#DBCLNP_RUN\"><code>DBCLNP_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
-        },
-        {
-          "language_code": "es_es",
-          "string": "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#DBCLNP_RUN\"><code>DBCLNP_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
-        },
-        {
-          "language_code": "de_de",
-          "string": "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#DBCLNP_RUN\"><code>DBCLNP_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
         }
       ]
     },
@@ -194,14 +169,6 @@
         {
           "language_code": "en_us",
           "string": "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted."
-        },
-        {
-          "language_code": "es_es",
-          "string": "Tiempo máximo en segundos para esperar a que finalice el script. Si se supera este tiempo, el script se cancela."
-        },
-        {
-          "language_code": "de_de",
-          "string": "Maximale Zeit in Sekunden, die auf den Abschluss des Skripts gewartet werden soll. Bei Überschreitung dieser Zeit wird das Skript abgebrochen."
         }
       ]
     },

front/plugins/db_cleanup/script.py

@@ -32,17 +32,9 @@ pluginName = 'DBCLNP'
 
 def main():
     
-    parser = argparse.ArgumentParser(description='DB cleanup tasks')
-    parser.add_argument('pluginskeephistory', action="store", help="TBC")
-    parser.add_argument('hourstokeepnewdevice', action="store", help="TBC")
-    parser.add_argument('daystokeepevents', action="store", help="TBC")
-    parser.add_argument('pholuskeepdays', action="store", help="TBC") # unused
-    
-    values = parser.parse_args()
-
-    PLUGINS_KEEP_HIST       = int(values.pluginskeephistory.split('=')[1])
-    HRS_TO_KEEP_NEWDEV      = int(values.hourstokeepnewdevice.split('=')[1])
-    DAYS_TO_KEEP_EVENTS     = int(values.daystokeepevents.split('=')[1])
+    PLUGINS_KEEP_HIST       = int(get_setting_value("PLUGINS_KEEP_HIST"))
+    HRS_TO_KEEP_NEWDEV      = int(get_setting_value("HRS_TO_KEEP_NEWDEV"))
+    DAYS_TO_KEEP_EVENTS     = int(get_setting_value("DAYS_TO_KEEP_EVENTS"))
     PHOLUS_DAYS_DATA        = get_setting_value("PHOLUS_DAYS_DATA")
     CLEAR_NEW_FLAG          = get_setting_value("CLEAR_NEW_FLAG")
 

front/plugins/maintenance/README.md

@@ -2,7 +2,7 @@
 
 A plugin responsible for general maintenance tasks. These currently include:
 
-- app.log cleanup
+- **`MAINT_LOG_LENGTH`**: app.log cleanup. Recommended value: `10000` lines. Increase if debugging an issue.
 
 ### Usage
 

server/device.py

@@ -550,7 +550,7 @@ def update_devices_names (db):
             recordsToUpdate.append ([newName, device['dev_MAC']])
 
     # Print log            
-    mylog('verbose', [f'[Update Device Name] Names Found (DiG/mDNS/NSLOOKUP/NBTSCAN/Pholus): {len(recordsToUpdate)} ({foundmDNSLookup}/{foundDig}/{foundNsLookup}/{foundNbtLookup}/{foundPholus})'] )                 
+    mylog('verbose', [f'[Update Device Name] Names Found (DiG/mDNS/NSLOOKUP/NBTSCAN/Pholus): {len(recordsToUpdate)} ({foundDig}/{foundmDNSLookup}/{foundNsLookup}/{foundNbtLookup}/{foundPholus})'] )                 
     mylog('verbose', [f'[Update Device Name] Names Not Found         : {notFound}'] )    
      
     # update not found devices with (name not found) 

server/plugin.py

@@ -441,7 +441,7 @@ def execute_plugin(db, all_plugins, plugin, pluginsState = plugins_state() ):
     
     # check if the subprocess / SQL query failed / there was no valid output
     if len(sqlParams) == 0: 
-        mylog('none', ['[Plugins] No output received from the plugin ', plugin["unique_prefix"], ' - enable LOG_LEVEL=debug and check logs'])
+        mylog('none', [f'[Plugins] No output received from the plugin "{plugin["unique_prefix"]}"'])
         return pluginsState 
     else: 
         mylog('verbose', ['[Plugins] SUCCESS, received ', len(sqlParams), ' entries'])