From 3390384ce3795eaf1e0cbc637c7d8d644c45b568 Mon Sep 17 00:00:00 2001 From: Ingo Ratsdorf Date: Wed, 10 Sep 2025 18:22:05 +1200 Subject: [PATCH 1/4] DB functions tidyup Added PRAGMAs for better DB performance on open. Integrated some Fake8 comments and eliminated some looping with more efficient pyton functions. --- server/database.py | 99 +++++++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/server/database.py b/server/database.py index 1fd13a1e..0ee12e90 100755 --- a/server/database.py +++ b/server/database.py @@ -1,18 +1,16 @@ """ all things database to support NetAlertX """ import sqlite3 -import base64 -import json # Register NetAlertX modules -from const import fullDbPath, sql_devices_stats, sql_devices_all, sql_generateGuid +from const import fullDbPath, sql_devices_stats, sql_devices_all from logger import mylog -from helper import timeNowTZ -from db.db_helper import row_to_json, get_table_json, json_obj +from db.db_helper import get_table_json, json_obj from workflows.app_events import AppEvent_obj from db.db_upgrade import ensure_column, ensure_views, ensure_CurrentScan, ensure_plugins_tables, ensure_Parameters, ensure_Settings, ensure_Indexes + class DB(): """ DB Class to provide the basic database interactions. @@ -24,28 +22,38 @@ class DB(): self.sql_connection = None #------------------------------------------------------------------------------- - def open (self): + def open(self): # Check if DB is open - if self.sql_connection != None : - mylog('debug','openDB: database already open') + if self.sql_connection is not None: + mylog('debug', 'openDB: database already open') return - mylog('verbose', '[Database] Opening DB' ) + mylog('verbose', '[Database] Opening DB') # Open DB and Cursor try: - self.sql_connection = sqlite3.connect (fullDbPath, isolation_level=None) - self.sql_connection.execute('pragma journal_mode=wal') # + self.sql_connection = sqlite3.connect(fullDbPath, isolation_level=None) + + # The WAL journaling mode uses a write-ahead log instead of a + # rollback journal to implement transactions. + self.sql_connection.execute('pragma journal_mode=WAL;') + # When synchronous is NORMAL (1), the SQLite database engine will still sync + # at the most critical moments, but less often than in FULL mode. + self.sql_connection.execute('PRAGMA synchronous=NORMAL;') + # When temp_store is MEMORY (2) temporary tables and indices + # are kept as if they were in pure in-memory databases. + self.sql_connection.execute('PRAGMA temp_store=MEMORY;') + self.sql_connection.text_factory = str self.sql_connection.row_factory = sqlite3.Row self.sql = self.sql_connection.cursor() except sqlite3.Error as e: - mylog('verbose',[ '[Database] - Open DB Error: ', e]) + mylog('minimal', ['[Database] - Open DB Error: ', e]) #------------------------------------------------------------------------------- - def commitDB (self): - if self.sql_connection == None : - mylog('debug','commitDB: database is not open') + def commitDB(self): + if self.sql_connection is None: + mylog('debug', 'commitDB: database is not open') return False # Commit changes to DB @@ -59,23 +67,19 @@ class DB(): #------------------------------------------------------------------------------- def get_sql_array(self, query): - if self.sql_connection == None : - mylog('debug','getQueryArray: database is not open') + if self.sql_connection is None: + mylog('debug', 'getQueryArray: database is not open') return self.sql.execute(query) rows = self.sql.fetchall() - #self.commitDB() + # self.commitDB() - # convert result into list of lists - arr = [] - for row in rows: - r_temp = [] - for column in row: - r_temp.append(column) - arr.append(r_temp) + # Convert result into list of lists + # Efficiently convert each row to a list + + return [list(row) for row in rows] - return arr #------------------------------------------------------------------------------- def initDB(self): @@ -84,39 +88,39 @@ class DB(): """ # Add Devices fields if missing - - # devFQDN + + # devFQDN if ensure_column(self.sql, "Devices", "devFQDN", "TEXT") is False: - return # addition failed + return # addition failed # devParentRelType if ensure_column(self.sql, "Devices", "devParentRelType", "TEXT") is False: - return # addition failed + return # addition failed # devRequireNicsOnline if ensure_column(self.sql, "Devices", "devReqNicsOnline", "INTEGER") is False: - return # addition failed - + return # addition failed + # Settings table setup ensure_Settings(self.sql) # Parameters tables setup ensure_Parameters(self.sql) - + # Plugins tables setup ensure_plugins_tables(self.sql) - + # CurrentScan table setup ensure_CurrentScan(self.sql) - - # Views + + # Views ensure_views(self.sql) - # Views + # Views ensure_Indexes(self.sql) # commit changes - self.commitDB() + self.commitDB() # Init the AppEvent database table AppEvent_obj(self) @@ -150,7 +154,7 @@ class DB(): try: result = get_table_json(self.sql, sqlQuery) except Exception as e: - mylog('verbose', ['[Database] - get_table_as_json ERROR:', e]) + mylog('minimal', ['[Database] - get_table_as_json ERROR:', e]) return json_obj({}, []) # return empty object on failure # mylog('debug',[ '[Database] - get_table_as_json - returning ', len(rows), " rows with columns: ", columnNames]) @@ -171,13 +175,13 @@ class DB(): rows = self.sql.fetchall() return rows except AssertionError: - mylog('verbose',[ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args]) + mylog('minimal', [ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args]) except sqlite3.Error as e: - mylog('verbose',[ '[Database] - SQL ERROR: ', e]) + mylog('minimal', [ '[Database] - SQL ERROR: ', e]) return None def read_one(self, query, *args): - """ + """ call read() with the same arguments but only returns the first row. should only be used when there is a single row result expected """ @@ -186,8 +190,8 @@ class DB(): rows = self.read(query, *args) if len(rows) == 1: return rows[0] - - if len(rows) > 1: + + if len(rows) > 1: mylog('verbose',[ '[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args]) return rows[0] # empty result set @@ -199,19 +203,22 @@ class DB(): def get_device_stats(db): # columns = ["online","down","all","archived","new","unknown"] return db.read_one(sql_devices_stats) + + #------------------------------------------------------------------------------- def get_all_devices(db): return db.read(sql_devices_all) + #------------------------------------------------------------------------------- - + def get_array_from_sql_rows(rows): # Convert result into list of lists arr = [] for row in rows: if isinstance(row, sqlite3.Row): arr.append(list(row)) # Convert row to list - elif isinstance(row, (tuple, list)): + elif isinstance(row, (tuple, list)): arr.append(list(row)) # Already iterable, just convert to list else: arr.append([row]) # Handle single values safely From 1874a5e641e9b842ee68ab13fcdb7d12dc181a43 Mon Sep 17 00:00:00 2001 From: Ingo Ratsdorf Date: Thu, 11 Sep 2025 10:24:55 +1200 Subject: [PATCH 2/4] CodeRabbit suggestionns Added some of the hand picked suggestions, including some outside of the previous changes. Some will improve documentation, some readability and some will affect performance. --- server/database.py | 188 +++++++++++++++++++++++++++++++-------------- 1 file changed, 129 insertions(+), 59 deletions(-) diff --git a/server/database.py b/server/database.py index 0ee12e90..8948ee1c 100755 --- a/server/database.py +++ b/server/database.py @@ -8,7 +8,10 @@ from const import fullDbPath, sql_devices_stats, sql_devices_all from logger import mylog from db.db_helper import get_table_json, json_obj from workflows.app_events import AppEvent_obj -from db.db_upgrade import ensure_column, ensure_views, ensure_CurrentScan, ensure_plugins_tables, ensure_Parameters, ensure_Settings, ensure_Indexes +from db.db_upgrade import ensure_column, \ + ensure_views, ensure_CurrentScan, \ + ensure_plugins_tables, ensure_Parameters, \ + ensure_Settings, ensure_Indexes class DB(): @@ -18,26 +21,50 @@ class DB(): """ def __init__(self): + """ + Initializes the class instance by setting up placeholders for the + SQL engine and SQL connection. + + Attributes: + sql: Placeholder for the SQL engine or session object. + sql_connection: Placeholder for the SQL database connection. + """ self.sql = None self.sql_connection = None - #------------------------------------------------------------------------------- def open(self): + """ + Opens a connection to the SQLite database if it is not already open. + This method initializes the database connection and cursor, and sets + several SQLite PRAGMA options to optimize performance and reliability: + - Enables Write-Ahead Logging (WAL) mode. + - Sets synchronous mode to NORMAL for a balance between + performance and safety. + - Stores temporary tables and indices in memory. + If the database is already open, the method logs a debug message + and returns. + If an error occurs during connection, it logs the error + with minimal verbosity. + Raises: + sqlite3.Error: If there is an error opening the database. + """ # Check if DB is open if self.sql_connection is not None: - mylog('debug', 'openDB: database already open') + mylog('debug', ['[Database] - open: DB already open']) return mylog('verbose', '[Database] Opening DB') # Open DB and Cursor try: - self.sql_connection = sqlite3.connect(fullDbPath, isolation_level=None) + self.sql_connection = sqlite3.connect(fullDbPath, + isolation_level=None) # The WAL journaling mode uses a write-ahead log instead of a # rollback journal to implement transactions. self.sql_connection.execute('pragma journal_mode=WAL;') - # When synchronous is NORMAL (1), the SQLite database engine will still sync - # at the most critical moments, but less often than in FULL mode. + # When synchronous is NORMAL (1), the SQLite database engine will + # still sync at the most critical moments, + # but less often than in FULL mode. self.sql_connection.execute('PRAGMA synchronous=NORMAL;') # When temp_store is MEMORY (2) temporary tables and indices # are kept as if they were in pure in-memory databases. @@ -49,9 +76,12 @@ class DB(): except sqlite3.Error as e: mylog('minimal', ['[Database] - Open DB Error: ', e]) - - #------------------------------------------------------------------------------- def commitDB(self): + """ + Commits the current transaction to the database. + Returns: + bool: True if the commit was successful, False if the database connection is not open. + """ if self.sql_connection is None: mylog('debug', 'commitDB: database is not open') return False @@ -60,13 +90,24 @@ class DB(): self.sql_connection.commit() return True - #------------------------------------------------------------------------------- def rollbackDB(self): + """ + Rolls back the current transaction in the database if a SQL connection exists. + + This method checks if a SQL connection is active and, if so, undoes all changes made in the current transaction, reverting the database to its previous state. + """ if self.sql_connection: self.sql_connection.rollback() - #------------------------------------------------------------------------------- def get_sql_array(self, query): + """ + Executes the given SQL query and returns the result as a list of lists. + Args: + query (str): The SQL query to execute. + Returns: + list[list]: A list of rows, where each row is represented as a list of column values. + Returns None if the database connection is not open. + """ if self.sql_connection is None: mylog('debug', 'getQueryArray: database is not open') return @@ -80,47 +121,57 @@ class DB(): return [list(row) for row in rows] - - #------------------------------------------------------------------------------- def initDB(self): """ - Check the current tables in the DB and upgrade them if neccessary + Initializes and upgrades the database schema for the application. + This method performs the following actions within a transaction: + - Ensures required columns exist in the 'Devices' table, adding them if missing. + - Sets up or updates the 'Settings', 'Parameters', 'Plugins', and 'CurrentScan' tables. + - Ensures necessary database views and indexes are present. + - Commits the transaction if all operations succeed. + - Rolls back the transaction and logs an error if any operation fails. + - Initializes the AppEvent database table after schema setup. + Raises: + RuntimeError: If ensuring any required column fails. + Exception: For any other errors encountered during initialization. """ - # Add Devices fields if missing + try: + # Start transactional upgrade + self.sql_connection.execute('BEGIN IMMEDIATE;') - # devFQDN - if ensure_column(self.sql, "Devices", "devFQDN", "TEXT") is False: - return # addition failed + # Add Devices fields if missing + if not ensure_column(self.sql, "Devices", "devFQDN", "TEXT"): + raise RuntimeError("ensure_column(devFQDN) failed") + if not ensure_column(self.sql, "Devices", "devParentRelType", "TEXT"): + raise RuntimeError("ensure_column(devParentRelType) failed") + if not ensure_column(self.sql, "Devices", "devReqNicsOnline", "INTEGER"): + raise RuntimeError("ensure_column(devReqNicsOnline) failed") - # devParentRelType - if ensure_column(self.sql, "Devices", "devParentRelType", "TEXT") is False: - return # addition failed + # Settings table setup + ensure_Settings(self.sql) - # devRequireNicsOnline - if ensure_column(self.sql, "Devices", "devReqNicsOnline", "INTEGER") is False: - return # addition failed + # Parameters tables setup + ensure_Parameters(self.sql) - # Settings table setup - ensure_Settings(self.sql) + # Plugins tables setup + ensure_plugins_tables(self.sql) - # Parameters tables setup - ensure_Parameters(self.sql) + # CurrentScan table setup + ensure_CurrentScan(self.sql) - # Plugins tables setup - ensure_plugins_tables(self.sql) + # Views + ensure_views(self.sql) - # CurrentScan table setup - ensure_CurrentScan(self.sql) + # Indexes + ensure_Indexes(self.sql) - # Views - ensure_views(self.sql) - - # Views - ensure_Indexes(self.sql) - - # commit changes - self.commitDB() + # commit changes + self.commitDB() + except Exception as e: + mylog('minimal', ['[Database] - initDB ERROR:', e]) + self.rollbackDB() # rollback any changes on error + raise # re-raise the exception # Init the AppEvent database table AppEvent_obj(self) @@ -185,47 +236,66 @@ class DB(): call read() with the same arguments but only returns the first row. should only be used when there is a single row result expected """ - - mylog('debug',[ '[Database] - Read One: ', query, " params: ", args]) + mylog('debug', ['[Database] - Read One: ', query, " params: ", args]) rows = self.read(query, *args) + if not rows: + return None if len(rows) == 1: return rows[0] - if len(rows) > 1: - mylog('verbose',[ '[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args]) + mylog('verbose', ['[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args]) return rows[0] # empty result set return None - -#------------------------------------------------------------------------------- def get_device_stats(db): + """ + Retrieve device statistics from the database. + + Args: + db: A database connection or handler object that provides a `read_one` method. + + Returns: + The result of the `read_one` method executed with the `sql_devices_stats` query, + typically containing statistics such as the number of devices online, down, all, + archived, new, or unknown. + + Raises: + Any exceptions raised by the underlying database handler. + """ # columns = ["online","down","all","archived","new","unknown"] return db.read_one(sql_devices_stats) -#------------------------------------------------------------------------------- def get_all_devices(db): + """ + Retrieve all devices from the database. + + Args: + db: A database connection or handler object that provides a `read` method. + + Returns: + The result of executing the `sql_devices_all` query using the database handler. + """ return db.read(sql_devices_all) -#------------------------------------------------------------------------------- - def get_array_from_sql_rows(rows): + """ + Converts a sequence of SQL query result rows into a list of lists. + Each row can be an instance of sqlite3.Row, a tuple, a list, or a single value. + - If the row is a sqlite3.Row, it is converted to a list. + - If the row is a tuple or list, it is converted to a list. + - If the row is a single value, it is wrapped in a list. + Args: + rows (Iterable): An iterable of rows returned from an SQL query. + Returns: + list: A list of lists, where each inner list represents a row of data. + """ # Convert result into list of lists - arr = [] - for row in rows: - if isinstance(row, sqlite3.Row): - arr.append(list(row)) # Convert row to list - elif isinstance(row, (tuple, list)): - arr.append(list(row)) # Already iterable, just convert to list - else: - arr.append([row]) # Handle single values safely + return [list(row) if isinstance(row, (sqlite3.Row, tuple, list)) else [row] for row in rows] - return arr - -#------------------------------------------------------------------------------- def get_temp_db_connection(): """ From de20a2621cef9d84184a73845799b36be2c1445f Mon Sep 17 00:00:00 2001 From: "Carlos V." <76731844+cvc90@users.noreply.github.com> Date: Thu, 11 Sep 2025 03:38:25 +0200 Subject: [PATCH 3/4] Update deviceDetailsTools.php Change static route to relative route in URL for proper proxy operation --- front/deviceDetailsTools.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/deviceDetailsTools.php b/front/deviceDetailsTools.php index 45d030db..2c043aa9 100755 --- a/front/deviceDetailsTools.php +++ b/front/deviceDetailsTools.php @@ -198,7 +198,7 @@
  • - +
    • From a823301862ec7984618d6a0f4950840be6209632 Mon Sep 17 00:00:00 2001 From: "Carlos V." <76731844+cvc90@users.noreply.github.com> Date: Thu, 11 Sep 2025 03:58:52 +0200 Subject: [PATCH 4/4] Update script.py Added user-agent header --- front/plugins/website_monitor/script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/plugins/website_monitor/script.py b/front/plugins/website_monitor/script.py index f3f35568..0426444a 100755 --- a/front/plugins/website_monitor/script.py +++ b/front/plugins/website_monitor/script.py @@ -57,7 +57,7 @@ def check_services_health(site): requests.packages.urllib3.disable_warnings(InsecureRequestWarning) try: - resp = requests.get(site, verify=False, timeout=get_setting_value('WEBMON_RUN_TIMEOUT')) + resp = requests.get(site, verify=False, timeout=get_setting_value('WEBMON_RUN_TIMEOUT'), headers={"User-Agent": "NetAlertX"}) latency = resp.elapsed.total_seconds() status = resp.status_code except SSLError: