From c1c6813b6e19803094cd9f38442a228dfba527e9 Mon Sep 17 00:00:00 2001 From: jokob-sk Date: Thu, 14 Nov 2024 16:50:23 +1100 Subject: [PATCH] GraphQl 0.123 - Dynamic columns + re-adding old Device table columns --- front/devices.php | 105 +++++++--------- front/php/server/query_graphql.php | 1 - server/const.py | 15 ++- server/database.py | 112 ++++++++++++++++-- server/graphql_server/graphql_schema.py | 38 ++++-- server/graphql_server/graphql_server_start.py | 12 +- server/helper.py | 37 ++++++ 7 files changed, 235 insertions(+), 85 deletions(-) diff --git a/front/devices.php b/front/devices.php index 93c305c3..c93da0d8 100755 --- a/front/devices.php +++ b/front/devices.php @@ -160,6 +160,7 @@ var tableColumnOrder = []; var tableColumnVisible = []; headersDefaultOrder = []; + missingNumbers = []; // Read parameters & Initialize components callAfterAppInitialized(main) @@ -193,7 +194,7 @@ function main () { const fullArray = Array.from({ length: tableColumnOrder.length }, (_, i) => i); // Filter out the elements already present in inputArray - const missingNumbers = fullArray.filter(num => !tableColumnVisible.includes(num)); + missingNumbers = fullArray.filter(num => !tableColumnVisible.includes(num)); // Concatenate the inputArray with the missingNumbers tableColumnOrder = [...tableColumnVisible, ...missingNumbers]; @@ -336,47 +337,43 @@ function filterDataByStatus(data, status) { }); } -// ----------------------------------------------------------------------------- -function getDeviceStatus(item) -{ - - if(item.devIsNew === 1) - { - return 'New'; - } - else if(item.devPresentLastScan === 1) - { - return 'On-line'; - } - else if(item.devPresentLastScan === 0 && item.devAlertDown !== 0) - { - return 'Down'; - } - else if(item.devIsArchived === 1) - { - return 'Archived'; - } - else if(item.devPresentLastScan === 0) - { - return 'Off-line'; - } - - return "Unknown status" -} // Map column index to column name for GraphQL query -function mapColumnIndexToFieldName(index) { +function mapColumnIndexToFieldName(index, tableColumnVisible) { const columnNames = [ - "rowid", "devMac", "devName", "devOwner", "devType", "devVendor", - "devFavorite", "devGroup", "devComments", "devFirstConnection", - "devLastConnection", "devLastIP", "devStaticIP", "devScan", "devLogEvents", - "devAlertEvents", "devAlertDown", "devSkipRepeated", "devLastNotification", - "devPresentLastScan", "devIsNew", "devLocation", "devIsArchived", - "devParentMAC", "devParentPort", "devIcon", "devGUID", "devSite", "devSSID", - "devSyncHubNode", "devSourcePlugin" + "devName", + "devOwner", + "devType", + "devIcon", + "devFavorite", + "devGroup", + "devFirstConnection", + "devLastConnection", + "devLastIP", + "devIsRandomMac", // resolved on the fly + "devStatus", // resolved on the fly + "devMac", + "devIpLong", //formatIPlong(device.devLastIP) || "", // IP orderable + "rowid", + "devParentMAC", + "devParentChildrenCount", // resolved on the fly + "devLocation", + "devVendor", + "devParentPort", + "devGUID", + "devSyncHubNode", + "devSite", + "devSSID", + "devSourcePlugin" ]; - return columnNames[index] || null; + console.log(index); + console.log(tableColumnVisible); + console.log(tableColumnOrder); // this + console.log(missingNumbers); + console.log(columnNames[tableColumnOrder[index]]); + + return columnNames[tableColumnOrder[index]] || null; } @@ -465,6 +462,7 @@ function initializeDatatable (status) { devLastNotification devPresentLastScan devIsNew + devIsRandomMac devLocation devIsArchived devParentMAC @@ -475,6 +473,9 @@ function initializeDatatable (status) { devSSID devSyncHubNode devSourcePlugin + devStatus + devParentChildrenCount + devIpLong } count } @@ -493,7 +494,7 @@ function initializeDatatable (status) { "page": Math.floor(d.start / d.length) + 1, // Page number (1-based) "limit": parseInt(d.length, 10), // Page size (ensure it's an integer) "sort": d.order && d.order[0] ? [{ - "field": mapColumnIndexToFieldName(d.order[0].column), // Sort field from DataTable column + "field": mapColumnIndexToFieldName(d.order[0].column, tableColumnVisible), // Sort field from DataTable column "order": d.order[0].dir.toUpperCase() // Sort direction (ASC/DESC) }] : [], // Default to an empty array if no sorting is defined "search": d.search.value // Search query @@ -518,13 +519,13 @@ function initializeDatatable (status) { device.devFirstConnection || "", device.devLastConnection || "", device.devLastIP || "", - (isRandomMAC(device.devMac)) || "", // Custom logic for randomized MAC - getDeviceStatus(device) || "", + device.devIsRandomMac || "", // Custom logic for randomized MAC + device.devStatus || "", device.devMac || "", // hidden - formatIPlong(device.devLastIP) || "", // IP orderable + device.devIpLong || "", // IP orderable device.rowid || "", device.devParentMAC || "", - getNumberOfChildren(device.devMac, json.devices.devices) || 0, + device.devParentChildrenCount || 0, device.devLocation || "", device.devVendor || "", device.devParentPort || 0, @@ -751,26 +752,6 @@ function initializeDatatable (status) { } - - - -// ----------------------------------------------------------------------------- -function getNumberOfChildren(mac, devices) -{ - childrenCount = 0; - - $.each(devices, function(index, dev) { - - if(dev.devParentMAC != null && dev.devParentMAC.trim() == mac.trim()) - { - childrenCount++; - } - - }); - - return childrenCount; -} - // ----------------------------------------------------------------------------- function handleLoadingDialog(needsReload = false) { diff --git a/front/php/server/query_graphql.php b/front/php/server/query_graphql.php index bc13d972..8c733134 100755 --- a/front/php/server/query_graphql.php +++ b/front/php/server/query_graphql.php @@ -12,7 +12,6 @@ require dirname(__FILE__).'/../server/init.php'; // Helper function to get GraphQL URL (you can replace this with environment variables) function getGraphQLUrl() { $port = getSettingValue("GRAPHQL_PORT"); // Port for the GraphQL server - // return "$url:$port/graphql"; // Full URL to the GraphQL endpoint return "0.0.0.0:$port/graphql"; // Full URL to the GraphQL endpoint } diff --git a/server/const.py b/server/const.py index de9b7fed..65becf44 100755 --- a/server/const.py +++ b/server/const.py @@ -27,7 +27,20 @@ vendorsPathNewest = '/usr/share/arp-scan/ieee-oui_all_filtered.txt' #=============================================================================== # SQL queries #=============================================================================== -sql_devices_all = """select rowid, * from Devices""" +sql_devices_all = """ + SELECT + rowid, + *, + CASE + WHEN devIsNew = 1 THEN 'New' + WHEN devPresentLastScan = 1 THEN 'On-line' + WHEN devPresentLastScan = 0 AND devAlertDown != 0 THEN 'Down' + WHEN devIsArchived = 1 THEN 'Archived' + WHEN devPresentLastScan = 0 THEN 'Off-line' + ELSE 'Unknown status' + END AS devStatus + FROM Devices + """ sql_appevents = """select * from AppEvents""" sql_devices_stats = """SELECT Online_Devices as online, Down_Devices as down, All_Devices as 'all', Archived_Devices as archived, (select count(*) from Devices a where devIsNew = 1 ) as new, diff --git a/server/database.py b/server/database.py index cc58b521..2ff34aaf 100755 --- a/server/database.py +++ b/server/database.py @@ -28,7 +28,7 @@ class DB(): mylog('debug','openDB: database already open') return - mylog('none', '[Database] Opening DB' ) + mylog('verbose', '[Database] Opening DB' ) # Open DB and Cursor try: self.sql_connection = sqlite3.connect (fullDbPath, isolation_level=None) @@ -37,7 +37,7 @@ class DB(): self.sql_connection.row_factory = sqlite3.Row self.sql = self.sql_connection.cursor() except sqlite3.Error as e: - mylog('none',[ '[Database] - Open DB Error: ', e]) + mylog('verbose',[ '[Database] - Open DB Error: ', e]) #------------------------------------------------------------------------------- @@ -96,7 +96,7 @@ class DB(): """) # ------------------------------------------------------------------- - # DevicesNew - cleanup after 6/6/2025 + # DevicesNew - cleanup after 6/6/2025 - need to update also DB in the source code! # check if migration already done based on devMac devMac_missing = self.sql.execute (""" @@ -104,6 +104,104 @@ class DB(): """).fetchone()[0] == 0 if devMac_missing: + + # ------------------------------------------------------------------------- + # Alter Devices table + # ------------------------------------------------------------------------- + # dev_Network_Node_MAC_ADDR column + dev_Network_Node_MAC_ADDR_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_MAC_ADDR' + """).fetchone()[0] == 0 + + if dev_Network_Node_MAC_ADDR_missing : + mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_MAC_ADDR to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_Network_Node_MAC_ADDR" TEXT + """) + + # dev_Network_Node_port column + dev_Network_Node_port_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_port' + """).fetchone()[0] == 0 + + if dev_Network_Node_port_missing : + mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_port to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_Network_Node_port" INTEGER + """) + + # dev_Icon column + dev_Icon_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Icon' + """).fetchone()[0] == 0 + + if dev_Icon_missing : + mylog('verbose', ["[upgradeDB] Adding dev_Icon to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_Icon" TEXT + """) + + # dev_GUID column + dev_GUID_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_GUID' + """).fetchone()[0] == 0 + + if dev_GUID_missing : + mylog('verbose', ["[upgradeDB] Adding dev_GUID to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_GUID" TEXT + """) + + # dev_NetworkSite column + dev_NetworkSite_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_NetworkSite' + """).fetchone()[0] == 0 + + if dev_NetworkSite_missing : + mylog('verbose', ["[upgradeDB] Adding dev_NetworkSite to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_NetworkSite" TEXT + """) + + # dev_SSID column + dev_SSID_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_SSID' + """).fetchone()[0] == 0 + + if dev_SSID_missing : + mylog('verbose', ["[upgradeDB] Adding dev_SSID to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_SSID" TEXT + """) + + # SQL query to update missing dev_GUID + self.sql.execute(f''' + UPDATE Devices + SET dev_GUID = {sql_generateGuid} + WHERE dev_GUID IS NULL + ''') + + # dev_SyncHubNodeName column + dev_SyncHubNodeName_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_SyncHubNodeName' + """).fetchone()[0] == 0 + + if dev_SyncHubNodeName_missing : + mylog('verbose', ["[upgradeDB] Adding dev_SyncHubNodeName to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_SyncHubNodeName" TEXT + """) + + # dev_SourcePlugin column + dev_SourcePlugin_missing = self.sql.execute (""" + SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_SourcePlugin' + """).fetchone()[0] == 0 + + if dev_SourcePlugin_missing : + mylog('verbose', ["[upgradeDB] Adding dev_SourcePlugin to the Devices table"]) + self.sql.execute(""" + ALTER TABLE "Devices" ADD "dev_SourcePlugin" TEXT + """) # SQL to create Devices table with indexes sql_create_devices_new_tmp = """ @@ -743,7 +841,7 @@ class DB(): columnNames = list(map(lambda x: x[0], self.sql.description)) rows = self.sql.fetchall() except sqlite3.Error as e: - mylog('none',[ '[Database] - SQL ERROR: ', e]) + mylog('verbose',[ '[Database] - SQL ERROR: ', e]) return json_obj({}, []) # return empty object result = {"data":[]} @@ -768,9 +866,9 @@ class DB(): rows = self.sql.fetchall() return rows except AssertionError: - mylog('none',[ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args]) + mylog('verbose',[ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args]) except sqlite3.Error as e: - mylog('none',[ '[Database] - SQL ERROR: ', e]) + mylog('verbose',[ '[Database] - SQL ERROR: ', e]) return None def read_one(self, query, *args): @@ -785,7 +883,7 @@ class DB(): return rows[0] if len(rows) > 1: - mylog('none',[ '[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args]) + mylog('verbose',[ '[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args]) return rows[0] # empty result set return None diff --git a/server/graphql_server/graphql_schema.py b/server/graphql_server/graphql_schema.py index bd0a6035..8ce7af19 100755 --- a/server/graphql_server/graphql_schema.py +++ b/server/graphql_server/graphql_schema.py @@ -9,6 +9,7 @@ sys.path.extend([f"{INSTALL_PATH}/server"]) from logger import mylog from const import apiPath +from helper import is_random_mac, get_number_of_children, format_ip_long # Define a base URL with the user's home directory folder = apiPath @@ -57,6 +58,11 @@ class Device(ObjectType): devSSID = String() devSyncHubNode = String() devSourcePlugin = String() + devStatus = String() + devIsRandomMac = Int() + devParentChildrenCount = Int() + devIpLong = Int() + class DeviceResult(ObjectType): devices = List(Device) @@ -67,6 +73,7 @@ class Query(ObjectType): devices = Field(DeviceResult, options=PageQueryOptionsInput()) def resolve_devices(self, info, options=None): + mylog('none', f'[graphql_schema] resolve_devices: {self}') try: with open(folder + 'table_devices.json', 'r') as f: devices_data = json.load(f)["data"] @@ -74,16 +81,19 @@ class Query(ObjectType): mylog('none', f'[graphql_schema] Error loading devices data: {e}') return DeviceResult(devices=[], count=0) + + # Add dynamic fields to each device + for device in devices_data: + device["devIsRandomMac"] = 1 if is_random_mac(device["devMac"]) else 0 + device["devParentChildrenCount"] = get_number_of_children(device["devMac"], devices_data) + device["devIpLong"] = format_ip_long(device.get("devLastIP", "")) + total_count = len(devices_data) - # Apply pagination and sorting if options are provided - if options: - # Implement pagination and sorting here - if options.page and options.limit: - start = (options.page - 1) * options.limit - end = start + options.limit - devices_data = devices_data[start:end] + mylog('none', f'[graphql_schema] devices_data: {devices_data}') + # Apply sorting if options are provided + if options: if options.sort: for sort_option in options.sort: devices_data = sorted( @@ -91,7 +101,7 @@ class Query(ObjectType): key=lambda x: x.get(sort_option.field), reverse=(sort_option.order.lower() == "desc") ) - + # Filter data if a search term is provided if options.search: devices_data = [ @@ -99,7 +109,17 @@ class Query(ObjectType): if options.search.lower() in device.get("devName", "").lower() ] - return DeviceResult(devices=devices_data, count=total_count) + # Then apply pagination + if options.page and options.limit: + start = (options.page - 1) * options.limit + end = start + options.limit + devices_data = devices_data[start:end] + + # Convert dict objects to Device instances to enable field resolution + devices = [Device(**device) for device in devices_data] + + return DeviceResult(devices=devices, count=total_count) + # Schema Definition diff --git a/server/graphql_server/graphql_server_start.py b/server/graphql_server/graphql_server_start.py index ebb7455d..4174a0d5 100755 --- a/server/graphql_server/graphql_server_start.py +++ b/server/graphql_server/graphql_server_start.py @@ -12,7 +12,8 @@ INSTALL_PATH="/app" sys.path.extend([f"{INSTALL_PATH}/server"]) from logger import mylog -from helper import get_setting_value +from helper import get_setting_value, timeNowTZ +from notification import write_notification app = Flask(__name__) @@ -24,22 +25,23 @@ def graphql_endpoint(): # Check for API token in headers token = request.headers.get("Authorization") if token != f"Bearer {API_TOKEN}": - mylog('none', [f'[graphql_server] Unauthorized access attempt']) + mylog('verbose', [f'[graphql_server] Unauthorized access attempt']) + return jsonify({"error": "Unauthorized"}), 401 data = request.get_json() - mylog('none', [f'[graphql_server] data: {data}']) + mylog('verbose', [f'[graphql_server] data: {data}']) + # Use the schema to execute the GraphQL query result = devicesSchema.execute(data.get("query"), variables=data.get("variables")) - mylog('none', [f'[graphql_server] result: {result}']) # Return the data from the query in JSON format return jsonify(result.data) def start_server(): """Function to start the GraphQL server in a background thread.""" - mylog('none', [f'[graphql_server] Starting on port "{GRAPHQL_PORT}"']) + mylog('verbose', [f'[graphql_server] Starting on port: {GRAPHQL_PORT}']) # Start the Flask app in a separate thread thread = threading.Thread(target=lambda: app.run(host="0.0.0.0", port=GRAPHQL_PORT, debug=True, use_reloader=False)) diff --git a/server/helper.py b/server/helper.py index c30812a5..8a681558 100755 --- a/server/helper.py +++ b/server/helper.py @@ -17,6 +17,7 @@ import base64 import hashlib import random import string +import ipaddress import conf @@ -911,6 +912,42 @@ def generate_random_string(length): characters = string.ascii_letters + string.digits return ''.join(random.choice(characters) for _ in range(length)) + +# Helper function to determine if a MAC address is random +def is_random_mac(mac): + # Check if second character matches "2", "6", "A", "E" (case insensitive) + is_random = mac[1].upper() in ["2", "6", "A", "E"] + + # Check against user-defined non-random MAC prefixes + if is_random: + not_random_prefixes = get_setting_value("UI_NOT_RANDOM_MAC") + for prefix in not_random_prefixes: + if mac.startswith(prefix): + is_random = False + break + return is_random + +# Helper function to calculate number of children +def get_number_of_children(mac, devices): + # Count children by checking devParentMAC for each device + return sum(1 for dev in devices if dev.get("devParentMAC", "").strip() == mac.strip()) + + + +# Function to convert IP to a long integer +def format_ip_long(ip_address): + try: + # Check if it's an IPv6 address + if ':' in ip_address: + ip = ipaddress.IPv6Address(ip_address) + else: + # Assume it's an IPv4 address + ip = ipaddress.IPv4Address(ip_address) + return int(ip) + except ValueError: + # Return a default error value if IP is invalid + return -1 + #------------------------------------------------------------------------------- # JSON methods #-------------------------------------------------------------------------------