From e899f657c5e4037eaf9117e2fae874e7f1e71581 Mon Sep 17 00:00:00 2001 From: jokob-sk Date: Tue, 10 Feb 2026 07:39:11 +1100 Subject: [PATCH] BE+FE: refactor totals retrieval + LUCIRPC old field name Signed-off-by: jokob-sk --- front/plugins/luci_import/config.json | 2 +- front/plugins/vendor_update/script.py | 25 +++++++-------- server/api_server/graphql_endpoint.py | 33 +++++++++++++++----- server/const.py | 26 +++++++++++++--- server/db/authoritative_handler.py | 7 ++--- server/db/db_helper.py | 39 ++++++++++++++--------- server/db/db_upgrade.py | 4 +-- server/models/device_instance.py | 45 ++++++++++++++++++++------- server/scan/device_handling.py | 30 +++++++++--------- server/scan/session_events.py | 3 +- 10 files changed, 142 insertions(+), 72 deletions(-) diff --git a/front/plugins/luci_import/config.json b/front/plugins/luci_import/config.json index 8b25c24d..e12eaeac 100755 --- a/front/plugins/luci_import/config.json +++ b/front/plugins/luci_import/config.json @@ -529,7 +529,7 @@ }, { "column": "Watched_Value2", - "mapped_to_column": "cur_NAME", + "mapped_to_column": "scanName", "css_classes": "col-sm-2", "show": true, "type": "label", diff --git a/front/plugins/vendor_update/script.py b/front/plugins/vendor_update/script.py index 78ad36b4..0ea4a62a 100755 --- a/front/plugins/vendor_update/script.py +++ b/front/plugins/vendor_update/script.py @@ -10,8 +10,8 @@ sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"]) from plugin_helper import Plugin_Objects, handleEmpty # noqa: E402 [flake8 lint suppression] from logger import mylog, Logger # noqa: E402 [flake8 lint suppression] -from helper import get_setting_value # noqa: E402 [flake8 lint suppression] -from const import logPath, applicationPath # noqa: E402 [flake8 lint suppression] +from helper import get_setting_value # noqa: E402 [flake8 lint suppression] +from const import logPath, applicationPath, NULL_EQUIVALENTS_SQL # noqa: E402 [flake8 lint suppression] from scan.device_handling import query_MAC_vendor # noqa: E402 [flake8 lint suppression] import conf # noqa: E402 [flake8 lint suppression] from pytz import timezone # noqa: E402 [flake8 lint suppression] @@ -83,17 +83,16 @@ def update_vendors(plugin_objects): mylog('verbose', [' Searching devices vendor']) # Get devices without a vendor - cursor.execute("""SELECT - devMac, - devLastIP, - devName, - devVendor - FROM Devices - WHERE devVendor = '(unknown)' - OR devVendor = '(Unknown)' - OR devVendor = '' - OR devVendor IS NULL - """) + query = f""" + SELECT + devMac, + devLastIP, + devName, + devVendor + FROM Devices + WHERE devVendor IN ({NULL_EQUIVALENTS_SQL}) OR devVendor IS NULL + """ + cursor.execute(query) devices = cursor.fetchall() conn.commit() diff --git a/server/api_server/graphql_endpoint.py b/server/api_server/graphql_endpoint.py index 0f99dc33..c7edb4fb 100755 --- a/server/api_server/graphql_endpoint.py +++ b/server/api_server/graphql_endpoint.py @@ -11,7 +11,7 @@ INSTALL_PATH = os.getenv("NETALERTX_APP", "/app") sys.path.extend([f"{INSTALL_PATH}/server"]) from logger import mylog # noqa: E402 [flake8 lint suppression] -from const import apiPath # noqa: E402 [flake8 lint suppression] +from const import apiPath, NULL_EQUIVALENTS # noqa: E402 [flake8 lint suppression] from helper import ( # noqa: E402 [flake8 lint suppression] is_random_mac, get_number_of_children, @@ -266,7 +266,7 @@ class Query(ObjectType): filtered.append(device) devices_data = filtered - + # 🔻 START If you change anything here, also update get_device_condition_by_status elif status == "connected": devices_data = [ device @@ -275,17 +275,17 @@ class Query(ObjectType): ] elif status == "favorites": devices_data = [ - device for device in devices_data if device["devFavorite"] == 1 + device for device in devices_data if device["devFavorite"] == 1 and device["devIsArchived"] == 0 ] elif status == "new": devices_data = [ - device for device in devices_data if device["devIsNew"] == 1 + device for device in devices_data if device["devIsNew"] == 1 and device["devIsArchived"] == 0 ] elif status == "down": devices_data = [ device for device in devices_data - if device["devPresentLastScan"] == 0 and device["devAlertDown"] + if device["devPresentLastScan"] == 0 and device["devAlertDown"] and device["devIsArchived"] == 0 ] elif status == "archived": devices_data = [ @@ -297,14 +297,33 @@ class Query(ObjectType): devices_data = [ device for device in devices_data - if device["devPresentLastScan"] == 0 + if device["devPresentLastScan"] == 0 and device["devIsArchived"] == 0 + ] + elif status == "unknown": + devices_data = [ + device + for device in devices_data + if device["devName"] in NULL_EQUIVALENTS and device["devIsArchived"] == 0 + ] + elif status == "known": + devices_data = [ + device + for device in devices_data + if device["devName"] not in NULL_EQUIVALENTS and device["devIsArchived"] == 0 ] elif status == "network_devices": devices_data = [ device for device in devices_data - if device["devType"] in network_dev_types + if device["devType"] in network_dev_types and device["devIsArchived"] == 0 ] + elif status == "network_devices_down": + devices_data = [ + device + for device in devices_data + if device["devType"] in network_dev_types and device["devPresentLastScan"] == 0 and device["devIsArchived"] == 0 + ] + # 🔺 END If you change anything here, also update get_device_condition_by_status elif status == "all_devices": devices_data = devices_data # keep all diff --git a/server/const.py b/server/const.py index 8725d51e..88f11a3e 100755 --- a/server/const.py +++ b/server/const.py @@ -49,6 +49,15 @@ NATIVE_SPEEDTEST_PATH = os.getenv("NATIVE_SPEEDTEST_PATH", "/usr/bin/speedtest") default_tz = "Europe/Berlin" +# =============================================================================== +# Magic strings +# =============================================================================== + +NULL_EQUIVALENTS = ["", "null", "(unknown)", "(Unknown)", "(name not found)"] + +# Convert list to SQL string: wrap each value in single quotes and escape single quotes if needed +NULL_EQUIVALENTS_SQL = ",".join(f"'{v.replace('\'', '\'\'')}'" for v in NULL_EQUIVALENTS) + # =============================================================================== # SQL queries @@ -186,10 +195,19 @@ sql_devices_filters = """ FROM Devices WHERE devSSID NOT IN ('', 'null') AND devSSID IS NOT NULL ORDER BY columnName; """ -sql_devices_stats = """SELECT Online_Devices as online, Down_Devices as down, All_Devices as 'all', Archived_Devices as archived, - (select count(*) from Devices a where devIsNew = 1 ) as new, - (select count(*) from Devices a where devName = '(unknown)' or devName = '(name not found)' ) as unknown - from Online_History order by Scan_Date desc limit 1""" + +sql_devices_stats = f""" + SELECT + Online_Devices as online, + Down_Devices as down, + All_Devices as 'all', + Archived_Devices as archived, + (SELECT COUNT(*) FROM Devices a WHERE devIsNew = 1) as new, + (SELECT COUNT(*) FROM Devices a WHERE devName IN ({NULL_EQUIVALENTS_SQL}) OR devName IS NULL) as unknown + FROM Online_History + ORDER BY Scan_Date DESC + LIMIT 1 + """ sql_events_pending_alert = "SELECT * FROM Events where eve_PendingAlertEmail is not 0" sql_settings = "SELECT * FROM Settings" sql_plugins_objects = "SELECT * FROM Plugins_Objects" diff --git a/server/db/authoritative_handler.py b/server/db/authoritative_handler.py index d40508c6..f0249e8e 100644 --- a/server/db/authoritative_handler.py +++ b/server/db/authoritative_handler.py @@ -19,6 +19,7 @@ from logger import mylog # noqa: E402 [flake8 lint suppression] from helper import get_setting_value # noqa: E402 [flake8 lint suppression] from db.db_helper import row_to_json # noqa: E402 [flake8 lint suppression] from plugin_helper import normalize_mac # noqa: E402 [flake8 lint suppression] +from const import NULL_EQUIVALENTS # noqa: E402 [flake8 lint suppression] # Map of field to its source tracking field @@ -96,7 +97,7 @@ def can_overwrite_field(field_name, current_value, current_source, plugin_prefix bool: True if overwrite allowed. """ - empty_values = ("0.0.0.0", "", "null", "(unknown)", "(name not found)", None) + empty_values = ("0.0.0.0", *NULL_EQUIVALENTS, None) # Rule 1: USER/LOCKED protected if current_source in ("USER", "LOCKED"): @@ -188,9 +189,7 @@ def get_source_for_field_update_with_value( if isinstance(field_value, str): stripped = field_value.strip() - if stripped in ("", "null"): - return "NEWDEV" - if stripped.lower() in ("(unknown)", "(name not found)"): + if stripped.lower() in NULL_EQUIVALENTS: return "NEWDEV" return plugin_prefix diff --git a/server/db/db_helper.py b/server/db/db_helper.py index 3d9bcc15..71a6ca6a 100755 --- a/server/db/db_helper.py +++ b/server/db/db_helper.py @@ -6,14 +6,34 @@ import os INSTALL_PATH = os.getenv("NETALERTX_APP", "/app") sys.path.extend([f"{INSTALL_PATH}/server"]) -from helper import if_byte_then_to_str # noqa: E402 [flake8 lint suppression] +from helper import if_byte_then_to_str, get_setting_value # noqa: E402 [flake8 lint suppression] from logger import mylog # noqa: E402 [flake8 lint suppression] +from const import NULL_EQUIVALENTS_SQL # noqa: E402 [flake8 lint suppression] + + +def get_device_conditions(): + network_dev_types = ",".join(f"'{v.replace('\'', '\'\'')}'" for v in get_setting_value("NETWORK_DEVICE_TYPES")) + + conditions = { + "all": "WHERE devIsArchived=0", + "my": "WHERE devIsArchived=0", + "connected": "WHERE devPresentLastScan=1", + "favorites": "WHERE devIsArchived=0 AND devFavorite=1", + "new": "WHERE devIsArchived=0 AND devIsNew=1", + "down": "WHERE devIsArchived=0 AND devAlertDown != 0 AND devPresentLastScan=0", + "offline": "WHERE devIsArchived=0 AND devPresentLastScan=0", + "archived": "WHERE devIsArchived=1", + "network_devices": f"WHERE devIsArchived=0 AND devType in ({network_dev_types})", + "network_devices_down": f"WHERE devIsArchived=0 AND devType in ({network_dev_types}) AND devPresentLastScan=0", + "unknown": f"WHERE devIsArchived=0 AND devName in ({NULL_EQUIVALENTS_SQL})", + "known": f"WHERE devIsArchived=0 AND devName not in ({NULL_EQUIVALENTS_SQL})", + } + + return conditions # ------------------------------------------------------------------------------- # Return the SQL WHERE clause for filtering devices based on their status. - - def get_device_condition_by_status(device_status): """ Return the SQL WHERE clause for filtering devices based on their status. @@ -32,17 +52,8 @@ def get_device_condition_by_status(device_status): str: SQL WHERE clause corresponding to the device status. Defaults to 'WHERE 1=0' for unrecognized statuses. """ - conditions = { - "all": "WHERE devIsArchived=0", - "my": "WHERE devIsArchived=0", - "connected": "WHERE devIsArchived=0 AND devPresentLastScan=1", - "favorites": "WHERE devIsArchived=0 AND devFavorite=1", - "new": "WHERE devIsArchived=0 AND devIsNew=1", - "down": "WHERE devIsArchived=0 AND devAlertDown != 0 AND devPresentLastScan=0", - "offline": "WHERE devIsArchived=0 AND devPresentLastScan=0", - "archived": "WHERE devIsArchived=1", - } - return conditions.get(device_status, "WHERE 1=0") + + return get_device_conditions().get(device_status, "WHERE 1=0") # ------------------------------------------------------------------------------- diff --git a/server/db/db_upgrade.py b/server/db/db_upgrade.py index 71e7fe5e..110dca17 100755 --- a/server/db/db_upgrade.py +++ b/server/db/db_upgrade.py @@ -147,7 +147,7 @@ def ensure_mac_lowercase_triggers(sql): except Exception as e: mylog("none", [f"[db_upgrade] ERROR while ensuring MAC triggers: {e}"]) return False - + def ensure_views(sql) -> bool: """ @@ -228,7 +228,7 @@ def ensure_views(sql) -> bool: ) SELECT d.*, -- all Device fields - r.* -- all CurrentScan fields (cur_*) + r.* -- all CurrentScan fields FROM Devices d LEFT JOIN RankedScans r ON d.devMac = r.scanMac diff --git a/server/models/device_instance.py b/server/models/device_instance.py index e522f4a9..b7f30c50 100755 --- a/server/models/device_instance.py +++ b/server/models/device_instance.py @@ -8,7 +8,7 @@ from front.plugins.plugin_helper import is_mac, normalize_mac from logger import mylog from models.plugin_object_instance import PluginObjectInstance from database import get_temp_db_connection -from db.db_helper import get_table_json, get_device_condition_by_status, row_to_json, get_date_from_period +from db.db_helper import get_table_json, get_device_conditions, get_device_condition_by_status, row_to_json, get_date_from_period from db.authoritative_handler import ( enforce_source_on_user_update, get_locked_field_overrides, @@ -331,22 +331,45 @@ class DeviceInstance: conn = get_temp_db_connection() sql = conn.cursor() - # Build a combined query with sub-selects for each status - query = f""" - SELECT - (SELECT COUNT(*) FROM Devices {get_device_condition_by_status("my")}) AS devices, - (SELECT COUNT(*) FROM Devices {get_device_condition_by_status("connected")}) AS connected, - (SELECT COUNT(*) FROM Devices {get_device_condition_by_status("favorites")}) AS favorites, - (SELECT COUNT(*) FROM Devices {get_device_condition_by_status("new")}) AS new, - (SELECT COUNT(*) FROM Devices {get_device_condition_by_status("down")}) AS down, - (SELECT COUNT(*) FROM Devices {get_device_condition_by_status("archived")}) AS archived - """ + conditions = get_device_conditions() + + # Build sub-selects dynamically for all dictionary entries + sub_queries = [] + for key, condition in conditions.items(): + # Make sure the alias is SQL-safe (no spaces or special chars) + alias = key.replace(" ", "_").lower() + sub_queries.append(f'(SELECT COUNT(*) FROM Devices {condition}) AS "{alias}"') + + # Join all sub-selects with commas + query = "SELECT\n " + ",\n ".join(sub_queries) sql.execute(query) row = sql.fetchone() conn.close() return list(row) if row else [] + def getNamedTotals(self): + """Get device totals by status.""" + conn = get_temp_db_connection() + sql = conn.cursor() + + conditions = get_device_conditions() + + # Build sub-selects dynamically for all dictionary entries + sub_queries = [] + for key, condition in conditions.items(): + # Make sure the alias is SQL-safe (no spaces or special chars) + alias = key.replace(" ", "_").lower() + sub_queries.append(f'(SELECT COUNT(*) FROM Devices {condition}) AS "{alias}"') + + # Join all sub-selects with commas + query = "SELECT\n " + ",\n ".join(sub_queries) + + mylog('none', [f'[getNamedTotals] query {query}']) + json_obj = get_table_json(sql, query, parameters=None) + + return json_obj + def getByStatus(self, status=None): """ Return devices filtered by status. Returns all if no status provided. diff --git a/server/scan/device_handling.py b/server/scan/device_handling.py index 886f550f..f9bebbba 100755 --- a/server/scan/device_handling.py +++ b/server/scan/device_handling.py @@ -5,7 +5,7 @@ import ipaddress from helper import get_setting_value, check_IP_format from utils.datetime_utils import timeNowDB, normalizeTimeStamp from logger import mylog, Logger -from const import vendorsPath, vendorsPathNewest, sql_generateGuid +from const import vendorsPath, vendorsPathNewest, sql_generateGuid, NULL_EQUIVALENTS from models.device_instance import DeviceInstance from scan.name_resolution import NameResolver from scan.device_heuristics import guess_icon, guess_type @@ -97,7 +97,7 @@ FIELD_SPECS = { "devName": { "scan_col": "scanName", "source_col": "devNameSource", - "empty_values": ["", "null", "(unknown)", "(name not found)"], + "empty_values": NULL_EQUIVALENTS, "default_value": "(unknown)", "priority": ["NSLOOKUP", "AVAHISCAN", "NBTSCAN", "DIGSCAN", "ARPSCAN", "DHCPLSS", "NEWDEV", "N/A"], }, @@ -108,7 +108,7 @@ FIELD_SPECS = { "devLastIP": { "scan_col": "scanLastIP", "source_col": "devLastIPSource", - "empty_values": ["", "null", "(unknown)", "(Unknown)"], + "empty_values": NULL_EQUIVALENTS, "priority": ["ARPSCAN", "NEWDEV", "N/A"], "default_value": "0.0.0.0", "allow_override_if_changed": True, @@ -120,7 +120,7 @@ FIELD_SPECS = { "devVendor": { "scan_col": "scanVendor", "source_col": "devVendorSource", - "empty_values": ["", "null", "(unknown)", "(Unknown)"], + "empty_values": NULL_EQUIVALENTS, "priority": ["VNDRPDT", "ARPSCAN", "NEWDEV", "N/A"], }, @@ -131,7 +131,7 @@ FIELD_SPECS = { "devSyncHubNode": { "scan_col": "scanSyncHubNode", "source_col": None, - "empty_values": ["", "null"], + "empty_values": NULL_EQUIVALENTS, "priority": None, }, @@ -141,7 +141,7 @@ FIELD_SPECS = { "devSite": { "scan_col": "scanSite", "source_col": None, - "empty_values": ["", "null"], + "empty_values": NULL_EQUIVALENTS, "priority": None, }, @@ -151,7 +151,7 @@ FIELD_SPECS = { "devVlan": { "scan_col": "scanVlan", "source_col": "devVlanSource", - "empty_values": ["", "null"], + "empty_values": NULL_EQUIVALENTS, "priority": None, }, @@ -161,7 +161,7 @@ FIELD_SPECS = { "devType": { "scan_col": "scanType", "source_col": None, - "empty_values": ["", "null"], + "empty_values": NULL_EQUIVALENTS, "priority": None, }, @@ -171,14 +171,14 @@ FIELD_SPECS = { "devParentMAC": { "scan_col": "scanParentMAC", "source_col": "devParentMACSource", - "empty_values": ["", "null"], + "empty_values": NULL_EQUIVALENTS, "priority": ["SNMPDSC", "UNIFIAPI", "UNFIMP", "NEWDEV", "N/A"], }, "devParentPort": { "scan_col": "scanParentPort", "source_col": None, - "empty_values": ["", "null"], + "empty_values": NULL_EQUIVALENTS, "priority": ["SNMPDSC", "UNIFIAPI", "UNFIMP", "NEWDEV", "N/A"], }, @@ -188,7 +188,7 @@ FIELD_SPECS = { "devSSID": { "scan_col": "scanSSID", "source_col": None, - "empty_values": ["", "null"], + "empty_values": NULL_EQUIVALENTS, "priority": ["SNMPDSC", "UNIFIAPI", "UNFIMP", "NEWDEV", "N/A"], }, } @@ -708,16 +708,16 @@ def create_new_devices(db): raw_name = str(scanName).strip() if scanName else "" raw_vendor = str(scanVendor).strip() if scanVendor else "" raw_ip = str(scanLastIP).strip() if scanLastIP else "" - if raw_ip.lower() in ("null", "(unknown)"): + if raw_ip.lower() in NULL_EQUIVALENTS: raw_ip = "" raw_ssid = str(scanSSID).strip() if scanSSID else "" - if raw_ssid.lower() in ("null", "(unknown)"): + if raw_ssid.lower() in NULL_EQUIVALENTS: raw_ssid = "" raw_parent_mac = str(scanParentMAC).strip() if scanParentMAC else "" - if raw_parent_mac.lower() in ("null", "(unknown)"): + if raw_parent_mac.lower() in NULL_EQUIVALENTS: raw_parent_mac = "" raw_parent_port = str(scanParentPort).strip() if scanParentPort else "" - if raw_parent_port.lower() in ("null", "(unknown)"): + if raw_parent_port.lower() in NULL_EQUIVALENTS: raw_parent_port = "" # Handle NoneType diff --git a/server/scan/session_events.py b/server/scan/session_events.py index 3f9f1022..116e23df 100755 --- a/server/scan/session_events.py +++ b/server/scan/session_events.py @@ -18,6 +18,7 @@ from utils.datetime_utils import timeNowDB from logger import mylog, Logger from messaging.reporting import skip_repeated_notifications from messaging.in_app import update_unread_notifications_count +from const import NULL_EQUIVALENTS, NULL_EQUIVALENTS_SQL # Make sure log level is initialized correctly @@ -222,7 +223,7 @@ def insert_events(db): FROM Devices, CurrentScan WHERE devMac = scanMac AND scanLastIP IS NOT NULL - AND scanLastIP NOT IN ('', 'null', '(unknown)', '(Unknown)') + AND scanLastIP NOT IN ({NULL_EQUIVALENTS_SQL}) AND scanLastIP <> COALESCE(devPrimaryIPv4, '') AND scanLastIP <> COALESCE(devPrimaryIPv6, '') AND scanLastIP <> COALESCE(devLastIP, '') """)