Merge branch 'main' of https://github.com/jokob-sk/NetAlertX

Restore manual naming of static leases using comments for MKSCAN + cleanups

.dockerignore

@@ -6,9 +6,6 @@
 docker-compose.yml
 Dockerfile
 Dockerfile.debian
-dockerfiles/LICENSE
-dockerfiles/README.md
-dockerfiles/README_ES.md
 docs
 LICENSE.txt
 README.md
@@ -16,4 +13,3 @@ CONTRIBUTING
 FUNDING.yml
 config/.gitignore
 db/.gitignore
-

Dockerfile

@@ -69,7 +69,7 @@ COPY --from=builder --chown=nginx:www-data ${INSTALL_DIR}/ ${INSTALL_DIR}/
 COPY install/crontab /etc/crontabs/root
 
 # Start all required services
-RUN ${INSTALL_DIR}/dockerfiles/pre-setup.sh
+RUN ${INSTALL_DIR}/dockerfiles/start.sh
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=2 \
     CMD curl -sf -o /dev/null ${LISTEN_ADDR}:${PORT}/php/server/query_json.php?file=app_state.json

README.md

@@ -8,8 +8,8 @@
 Get visibility of what's going on on your WIFI/LAN network. Schedule scans for devices, port changes and get alerts if unknown devices or changes are found. Write your own [Plugins](https://github.com/jokob-sk/NetAlertX/tree/main/front/plugins#readme) with auto-generated UI and in-build notification system. Build out and easily maintain your network source of truth (NSoT).
 
 
-| 🐳 [Docker hub](https://registry.hub.docker.com/r/jokobsk/netalertx) |  📑 [Docker guide](https://github.com/jokob-sk/NetAlertX/blob/main/dockerfiles/README.md) |🆕 [Release notes](https://github.com/jokob-sk/NetAlertX/releases) | 📚 [All Docs](https://github.com/jokob-sk/NetAlertX/tree/main/docs) | 🔌 [Plugins](https://github.com/jokob-sk/NetAlertX/blob/main/front/plugins/README.md) |
+| [📑 Docker guide](https://github.com/jokob-sk/NetAlertX/blob/main/dockerfiles/README.md) | [🚀 Releases](https://github.com/jokob-sk/NetAlertX/releases) | [📚 Docs](https://github.com/jokob-sk/NetAlertX/tree/main/docs) | [🔌 Plugins](https://github.com/jokob-sk/NetAlertX/blob/main/front/plugins/README.md) | [🤖 Ask AI](https://gurubase.io/g/netalertx)
-|----------------------|----------------------| ----------------------|  ----------------------| ----------------------| 
+|----------------------| ----------------------|  ----------------------| ----------------------| ----------------------| 
 
 ![showcase][showcase] 
 

dockerfiles/README.md

@@ -6,8 +6,8 @@
 
 # NetAlertX 🖧🔍 Network scanner & notification framework
 
-  | 🐳 [Docker hub](https://registry.hub.docker.com/r/jokobsk/netalertx) |  📑 [Docker guide](https://github.com/jokob-sk/NetAlertX/blob/main/dockerfiles/README.md) |🆕 [Release notes](https://github.com/jokob-sk/NetAlertX/releases) | 📚 [All Docs](https://github.com/jokob-sk/NetAlertX/tree/main/docs) | 🔌 [Plugins](https://github.com/jokob-sk/NetAlertX/blob/main/front/plugins/README.md) |
+| [📑 Docker guide](https://github.com/jokob-sk/NetAlertX/blob/main/dockerfiles/README.md) | [🚀 Releases](https://github.com/jokob-sk/NetAlertX/releases) | [📚 Docs](https://github.com/jokob-sk/NetAlertX/tree/main/docs) | [🔌 Plugins](https://github.com/jokob-sk/NetAlertX/blob/main/front/plugins/README.md) | [🤖 Ask AI](https://gurubase.io/g/netalertx)
-  |----------------------|----------------------| ----------------------|  ----------------------| ----------------------| 
+|----------------------| ----------------------|  ----------------------| ----------------------| ----------------------| 
 
 <a href="https://raw.githubusercontent.com/jokob-sk/NetAlertX/main/docs/img/GENERAL/github_social_image.jpg" target="_blank">
   <img src="https://raw.githubusercontent.com/jokob-sk/NetAlertX/main/docs/img/GENERAL/github_social_image.jpg" width="1000px" />

dockerfiles/init.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/with-contenv bash
 
 echo "---------------------------------------------------------"
-echo "[INSTALL]                                    Run setup.sh"
+echo "[INSTALL]                                    Run init.sh"
 echo "---------------------------------------------------------"
 
 export INSTALL_DIR=/app  # Specify the installation directory here

dockerfiles/start.sh

@@ -19,7 +19,7 @@ echo "oneshot" > /etc/s6-overlay/s6-rc.d/SetupOneshot/type
 echo "longrun" > /etc/s6-overlay/s6-rc.d/php-fpm/type
 echo "longrun" > /etc/s6-overlay/s6-rc.d/nginx/type
 echo "longrun" > /etc/s6-overlay/s6-rc.d/$APP_NAME/type
-echo -e "${INSTALL_DIR}/dockerfiles/setup.sh" > /etc/s6-overlay/s6-rc.d/SetupOneshot/up
+echo -e "${INSTALL_DIR}/dockerfiles/init.sh" > /etc/s6-overlay/s6-rc.d/SetupOneshot/up
 echo -e "#!/bin/execlineb -P\n/usr/sbin/php-fpm83 -F" > /etc/s6-overlay/s6-rc.d/php-fpm/run
 echo -e '#!/bin/execlineb -P\nnginx -g "daemon off;"' > /etc/s6-overlay/s6-rc.d/nginx/run
 echo -e '#!/bin/execlineb -P
@@ -38,6 +38,5 @@ touch /etc/s6-overlay/s6-rc.d/user/contents.d/{SetupOneshot,php-fpm,nginx,$APP_N
 touch /etc/s6-overlay/s6-rc.d/nginx/dependencies.d/php-fpm
 touch /etc/s6-overlay/s6-rc.d/$APP_NAME/dependencies.d/nginx
 
-
+# this removes the current file
-
+# rm -f $0
-rm -f $0

docs/BACKUPS.md

@@ -1,7 +1,8 @@
 # 💾 Backing things up
 
 > [!NOTE]
-> To backup 99% of your configuration backup at least the `/config` folder. Please read the whole page (or at least "Scenario 2: Corrupted database") for details.
+> To backup 99% of your configuration backup at least the `/app/config` folder. Please read the whole page (or at least "Scenario 2: Corrupted database") for details.
+> Please also note that database definitions might change over versions. The safest way is to restore your older backups into the **same version** of the app and then gradually upgarde between releases to the latest version.
 
 There are 3 artifacts that can be used to backup the application:
 
@@ -22,16 +23,17 @@ The core application configuration is in the `app.conf` file (See [Settings Syst
 - Notification settings
 - Scanner settings
 - Scheduled maintenance settings
-- UI configuration (80%)
+- UI configuration
 
 ### Core Device Data
 
-The core device data is backed up to the `devices_<timestamp>.csv` file via the [CSV Backup `CSVBCKP` Plugin](https://github.com/jokob-sk/NetAlertX/tree/main/front/plugins/csv_backup). This file contains data, such as:
+The core device data is backed up to the `devices_<timestamp>.csv` or `devices.csv` file via the [CSV Backup `CSVBCKP` Plugin](https://github.com/jokob-sk/NetAlertX/tree/main/front/plugins/csv_backup). This file contains data, such as:
 
 - Device names
-- Device Icons
+- Device icons
-- Device Network configuration
+- Device network configuration
 - Device categorization 
+- Device custom properties data
 
 ### Historical data
 
@@ -40,13 +42,13 @@ Historical data is stored in the `app.db` database (See [Database overview](http
 - Plugin objects
 - Plugin historical entries
 - History of Events, Notifications, Workflow Events
-- Presence History
+- Presence history
 
 ## 🧭 Backup strategies
 
 The safest approach to backups is to backup all of the above, by taking regular file system backups (I use [Kopia](https://github.com/kopia/kopia)). 
 
-Arguably, the most time is spent setting up the device list, so if only one file is kept I'd recommend to have a latest backup of the `devices_<timestamp>.csv` file, followed by the `app.conf` file. 
+Arguably, the most time is spent setting up the device list, so if only one file is kept I'd recommend to have a latest backup of the `devices_<timestamp>.csv` or `devices.csv` file, followed by the `app.conf` file. 
 
 ### Scenario 1: Full backup
 
@@ -54,8 +56,8 @@ End-result: Full restore
 
 #### Source artifacts:
 
-- `/db/app.db` (uncorrupted)
+- `/app/db/app.db` (uncorrupted)
-- `/config/app.conf`
+- `/app/config/app.conf`
 
 #### Recovery:
 
@@ -68,15 +70,15 @@ End-result: Partial restore (historical data & configurations from the Maintenan
 
 #### Source artifacts:
 
-- `/config/app.conf`
+- `/app/config/app.conf`
-- `/config/devices_<timestamp>.csv` or `/config/devices.csv`
+- `/app/config/devices_<timestamp>.csv` or `/app/config/devices.csv`
 
 #### Recovery:
 
 Even with a corrupted database you can recover what I would argue is 99% of the configuration. 
 
-- map the `/config/app.conf` file as described in the [Setup documentation](https://github.com/jokob-sk/NetAlertX/blob/main/dockerfiles/README.md#docker-paths).
+- upload the `app.conf` file into the mounted `/app/config/` folder as described in the [Setup documentation](https://github.com/jokob-sk/NetAlertX/blob/main/dockerfiles/README.md#docker-paths).
-- rename the `devices_<timestamp>.csv` to `devices.csv` and place it in the `/config` folder
+- rename the `devices_<timestamp>.csv` to `devices.csv` and place it in the `/app/config` folder
 - Restore the `devices.csv` backup via the [Maintenance section](https://github.com/jokob-sk/NetAlertX/blob/main/docs/DEVICES_BULK_EDITING.md)
 
 

docs/COMMUNITY_GUIDES.md

@@ -1,6 +1,6 @@
 # Community Guides
 
-Use the official installation guides at first and use community content as supplementary material. Open an issue if you'd like to add your link to the list 🙏 
+Use the official installation guides at first and use community content as supplementary material. Open an issue or PR if you'd like to add your link to the list 🙏 (Ordered by last update time)
 
 - ▶ [Home Lab Network Monitoring - Scotti-BYTE Enterprise Consulting Services](https://www.youtube.com/watch?v=0DryhzrQSJA) (July 2024)
 - 📄 [How to Install NetAlertX on Your Synology NAS - Marius hosting](https://mariushosting.com/how-to-install-pi-alert-on-your-synology-nas/) (Updated frequently)
@@ -12,5 +12,3 @@ Use the official installation guides at first and use community content as suppl
 - ▶ [Pi.Alert auf Synology & Docker by - Jürgen Barth](https://www.youtube.com/watch?v=-ouvA2UNu-A) (March 2023)
 - ▶ [Top Docker Container for Home Server Security - VirtualizationHowto](https://www.youtube.com/watch?v=tY-w-enLF6Q) (March 2023)
 - ▶ [Pi.Alert or WatchYourLAN can alert you to unknown devices appearing on your WiFi or LAN network - Danie van der Merwe](https://www.youtube.com/watch?v=v6an9QG2xF0) (November 2022)
-
-> Ordered by last update time. 

front/multiEditCore.php

@@ -81,7 +81,7 @@
         
         settingsData = res["data"];
 
-        excludedColumns = ["NEWDEV_devMac", "NEWDEV_devFirstConnection", "NEWDEV_devLastConnection", "NEWDEV_devLastNotification", "NEWDEV_devLastIP", "NEWDEV_devStaticIP", "NEWDEV_devScan", "NEWDEV_devPresentLastScan", "NEWDEV_devCustomProps" ]
+        excludedColumns = ["NEWDEV_devMac", "NEWDEV_devFirstConnection", "NEWDEV_devLastConnection", "NEWDEV_devLastNotification", "NEWDEV_devStaticIP", "NEWDEV_devScan", "NEWDEV_devPresentLastScan", "NEWDEV_devCustomProps" ]
         
         const relevantColumns = settingsData.filter(set =>
             set.setGroup === "NEWDEV" &&

front/php/templates/footer.php

@@ -28,11 +28,10 @@
     
     <!-- To the right -->
     <div class="pull-right no-hidden-xs">
-      | <a href="https://github.com/jokob-sk/NetAlertX/tree/main/docs#documentation-overview" target="_blank">Docs <i class="fa fa-circle-question"></i></a>
+      | <a href="https://gurubase.io/g/netalertx" target="_blank" title="Ask AI"><i class="fa-regular fa-comment-dots fa-flip-horizontal"></i></a> 
-      | <a href="https://github.com/jokob-sk/NetAlertX/issues"><i class="fa-solid fa-bug"></i></a> 
+      | <a href="https://github.com/jokob-sk/NetAlertX/tree/main/docs#documentation-overview" target="_blank" title="Documentation"><i class="fa fa-book"></i></a>
-      | <a href="https://github.com/jokob-sk/NetAlertX/"><i class="fa-brands fa-github"></i></a> 
+      | <a href="https://github.com/jokob-sk/NetAlertX/issues" target="_blank"><i class="fa-solid fa-bug" title="Report a bug"></i></a> 
-      | <a href="https://discord.gg/UQnnHNYV"><i class="fa-brands fa-discord"></i></a> 
+      | <a href="https://discord.com/invite/NczTUTWyRr" target="_blank"><i class="fa-brands fa-discord" title="Join Discord"></i></a> 
-      | <a href="mailto:netalertx@gmail.com?subject=NetAlertX"><i class="fa-solid fa-envelope"></i></a> 
       | <?= lang('Maintenance_built_on');?>:  <?php include 'php/templates/build.php'; ?> 
       |  Version:  <?php include 'php/templates/version.php'; ?> 
       |     

front/plugins/icmp_scan/config.json

@@ -131,6 +131,34 @@
         }
       ]
     },
+    {
+      "function": "IN_REGEX",
+      "type": {
+        "dataType": "string",
+        "elements": [
+          {
+            "elementType": "input",
+            "elementOptions": [],
+            "transformers": []
+          }
+        ]
+      },
+      "default_value": ".*",
+      "options": [],
+      "localized": ["name", "description"],
+      "name": [
+        {
+          "language_code": "en_us",
+          "string": "Inclusion REGEX"
+        }
+      ],
+      "description": [
+        {
+          "language_code": "en_us",
+          "string": "REGEX to specify which IPs are included in the scan. Examples: <code>192.168.1.*|10.0.0.1|172.16.5.*</code> (specific IPs or ranges). Use <code>.*</code> to include all IPs."
+        }
+      ]
+    },
     {
       "function": "RUN_SCHD",
       "type": {

front/plugins/icmp_scan/icmp.py

@@ -48,6 +48,7 @@ def main():
 
     timeout = get_setting_value('ICMP_RUN_TIMEOUT')
     args = get_setting_value('ICMP_ARGS')
+    in_regex = get_setting_value('ICMP_IN_REGEX')
  
     # Create a database connection
     db = DB()  # instance of class DB
@@ -62,9 +63,19 @@ def main():
     # Retrieve devices
     all_devices = device_handler.getAll()
 
-    mylog('verbose', [f'[{pluginName}] Devices to PING: {len(all_devices)}'])   
+    # Compile the regex for efficiency if it will be used multiple times
+    regex_pattern = re.compile(in_regex)
 
-    for device in all_devices:
+    # Filter devices based on the regex match
+    filtered_devices = [
+        device for device in all_devices 
+        if regex_pattern.match(device['devLastIP'])
+    ]
+
+
+    mylog('verbose', [f'[{pluginName}] Devices to PING: {len(filtered_devices)}'])   
+
+    for device in filtered_devices:
         is_online, output = execute_scan(device['devLastIP'], timeout, args)
 
         mylog('verbose', [f'[{pluginName}] ip: "{device['devLastIP']}" is_online: "{is_online}"'])

front/plugins/mikrotik_scan/config.json

@@ -2,7 +2,7 @@
   "code_name": "mikrotik_scan",
   "unique_prefix": "MTSCAN",
   "plugin_type": "device_scanner",
-  "execution_order" : "Layer_4",
+  "execution_order" : "Layer_1",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "CurrentScan",
@@ -27,12 +27,6 @@
     }
   ],
   "params": [
-    {
-      "name": "ips",
-      "type": "sql",
-      "value": "SELECT devLastIP from DEVICES order by devMac",
-      "timeoutMultiplier": true
-    },
     {
       "name": "mt_host",
       "type": "setting",

front/plugins/mikrotik_scan/mikrotik.py

@@ -1,27 +1,16 @@
 #!/usr/bin/env python
 
 import os
-import pathlib
-import argparse
-import subprocess
 import sys
-import hashlib
-import csv
-import sqlite3
-import re
-from io import StringIO
-from datetime import datetime
 
 # Register NetAlertX directories
 INSTALL_PATH="/app"
 sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"])
 
-from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
+from plugin_helper import Plugin_Objects
-from logger import mylog, Logger, append_line_to_file
+from logger import mylog, Logger
-from helper import timeNowTZ, get_setting_value
+from helper import get_setting_value
-from const import logPath, applicationPath, fullDbPath
+from const import logPath
-from database import DB
-from device import Device_obj
 import conf
 from pytz import timezone
 from librouteros import connect
@@ -81,6 +70,7 @@ def get_entries(plugin_objects: Plugin_Objects) -> Plugin_Objects:
             comment = lease.get('comment')
             last_seen = lease.get('last-seen')
             status = lease.get('status')
+            device_name = comment or host_name or "(unknown)"
     
             mylog('verbose', [f"ID: {lease_id}, Address: {address}, MAC Address: {mac_address}, Host Name: {host_name}, Comment: {comment}, Last Seen: {last_seen}, Status: {status}"])
 
@@ -89,9 +79,9 @@ def get_entries(plugin_objects: Plugin_Objects) -> Plugin_Objects:
                     primaryId   = mac_address,
                     secondaryId = address,
                     watched1    = address,
-                    watched2    = host_name,
+                    watched2    = device_name,
-                    watched3    = last_seen,
+                    watched3    = host_name,
-                    watched4    = '',
+                    watched4    = last_seen,
                     extra       = '',
                     helpVal1    = comment, 
                     foreignKey  = mac_address)

front/plugins/sync/sync.py

@@ -125,8 +125,6 @@ def main():
     # Mode 2: PULL/GET (HUB)
     
     # PULLING DEVICES 
-    
-    file_dir = os.path.join(pluginsPath, 'sync')
     file_prefix = 'last_result'
     
     # pull data from nodes if specified
@@ -145,7 +143,7 @@ def main():
             log_file_name = f'{file_prefix}.{node_name}.log'
 
             # Write decoded data to log file
-            with open(os.path.join(file_dir, log_file_name), 'wb') as log_file:
+            with open(os.path.join(LOG_PATH, log_file_name), 'wb') as log_file:
                 log_file.write(decoded_data)
 
             message = f'[{pluginName}] Device data from node "{node_name}" written to {log_file_name}'
@@ -157,7 +155,7 @@ def main():
     # Create the file path
 
     # Get all "last_result" files from the sync folder, decode, rename them, and get the list of files
-    files_to_process = decode_and_rename_files(file_dir, file_prefix)
+    files_to_process = decode_and_rename_files(LOG_PATH, file_prefix)
     
     if len(files_to_process) > 0:
                 
@@ -181,11 +179,11 @@ def main():
 
                 # Store e.g. Node_1 from last_result.encoded.Node_1.1.log
                 tmp_SyncHubNodeName = ''
-                if len(file_name.split('.')) > 3:
+                if len(file_name.split('.')) > 2:
-                    tmp_SyncHubNodeName = file_name.split('.')[2]   
+                    tmp_SyncHubNodeName = file_name.split('.')[1]   
 
 
-                file_path = f"{INSTALL_PATH}/front/plugins/sync/{file_name}"
+                file_path = f"{LOG_PATH}/{file_name}"
                 
                 with open(file_path, 'r') as f:
                     data = json.load(f)
@@ -197,7 +195,7 @@ def main():
                             
                 # Rename the file to "processed_" + current name
                 new_file_name = f"processed_{file_name}"
-                new_file_path = os.path.join(file_dir, new_file_name)
+                new_file_path = os.path.join(LOG_PATH, new_file_name)
 
                 # Overwrite if the new file already exists
                 if os.path.exists(new_file_path):

front/plugins/unifi_import/script.py

@@ -213,7 +213,7 @@ def get_unifi_val(obj, key, default='null'):
                 if result not in ['','None', None, 'null']:
                     return result
     
-    mylog('debug', [f'[{pluginName}] Value not found for key "{key}" in obj "{json.dumps(obj)}"'])
+    mylog('trace', [f'[{pluginName}] Value not found for key "{key}" in obj "{json.dumps(obj)}"'])
     return default