vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2026-04-14 06:01:34 -07:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

DB functions tidyup

Browse Source 
Added PRAGMAs for better DB performance on open. Integrated some Fake8 comments and eliminated some looping with more efficient pyton functions.
This commit is contained in:
Ingo Ratsdorf
2025-09-10 18:22:05 +12:00
parent 2b2ae516da
commit 3390384ce3
1 changed files with 53 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
server/database.py
View File

@@ -1,18 +1,16 @@
""" all things database to support NetAlertX """ """ all things database to support NetAlertX """
import sqlite3 import sqlite3
import base64
import json
# Register NetAlertX modules # Register NetAlertX modules
from const import fullDbPath, sql_devices_stats, sql_devices_all, sql_generateGuid from const import fullDbPath, sql_devices_stats, sql_devices_all
from logger import mylog from logger import mylog
from helper import timeNowTZ from db.db_helper import get_table_json, json_obj
from db.db_helper import row_to_json, get_table_json, json_obj
from workflows.app_events import AppEvent_obj from workflows.app_events import AppEvent_obj
from db.db_upgrade import ensure_column, ensure_views, ensure_CurrentScan, ensure_plugins_tables, ensure_Parameters, ensure_Settings, ensure_Indexes from db.db_upgrade import ensure_column, ensure_views, ensure_CurrentScan, ensure_plugins_tables, ensure_Parameters, ensure_Settings, ensure_Indexes
class DB(): class DB():
""" """
DB Class to provide the basic database interactions. DB Class to provide the basic database interactions.
@@ -24,28 +22,38 @@ class DB():
self.sql_connection = None self.sql_connection = None
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
def open (self): def open(self):
# Check if DB is open # Check if DB is open
if self.sql_connection != None : if self.sql_connection is not None:
mylog('debug','openDB: database already open') mylog('debug', 'openDB: database already open')
return return
mylog('verbose', '[Database] Opening DB' ) mylog('verbose', '[Database] Opening DB')
# Open DB and Cursor # Open DB and Cursor
try: try:
self.sql_connection = sqlite3.connect (fullDbPath, isolation_level=None) self.sql_connection = sqlite3.connect(fullDbPath, isolation_level=None)
self.sql_connection.execute('pragma journal_mode=wal') #
# The WAL journaling mode uses a write-ahead log instead of a
# rollback journal to implement transactions.
self.sql_connection.execute('pragma journal_mode=WAL;')
# When synchronous is NORMAL (1), the SQLite database engine will still sync
# at the most critical moments, but less often than in FULL mode.
self.sql_connection.execute('PRAGMA synchronous=NORMAL;')
# When temp_store is MEMORY (2) temporary tables and indices
# are kept as if they were in pure in-memory databases.
self.sql_connection.execute('PRAGMA temp_store=MEMORY;')
self.sql_connection.text_factory = str self.sql_connection.text_factory = str
self.sql_connection.row_factory = sqlite3.Row self.sql_connection.row_factory = sqlite3.Row
self.sql = self.sql_connection.cursor() self.sql = self.sql_connection.cursor()
except sqlite3.Error as e: except sqlite3.Error as e:
mylog('verbose',[ '[Database] - Open DB Error: ', e]) mylog('minimal', ['[Database] - Open DB Error: ', e])
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
def commitDB (self): def commitDB(self):
if self.sql_connection == None : if self.sql_connection is None:
mylog('debug','commitDB: database is not open') mylog('debug', 'commitDB: database is not open')
return False return False
# Commit changes to DB # Commit changes to DB
@@ -59,23 +67,19 @@ class DB():
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
def get_sql_array(self, query): def get_sql_array(self, query):
if self.sql_connection == None : if self.sql_connection is None:
mylog('debug','getQueryArray: database is not open') mylog('debug', 'getQueryArray: database is not open')
return return
self.sql.execute(query) self.sql.execute(query)
rows = self.sql.fetchall() rows = self.sql.fetchall()
#self.commitDB() # self.commitDB()
# convert result into list of lists # Convert result into list of lists
arr = [] # Efficiently convert each row to a list
for row in rows:
r_temp = [] return [list(row) for row in rows]
for column in row:
r_temp.append(column)
arr.append(r_temp)
return arr
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
def initDB(self): def initDB(self):
@@ -84,39 +88,39 @@ class DB():
""" """
# Add Devices fields if missing # Add Devices fields if missing
# devFQDN # devFQDN
if ensure_column(self.sql, "Devices", "devFQDN", "TEXT") is False: if ensure_column(self.sql, "Devices", "devFQDN", "TEXT") is False:
return # addition failed return # addition failed
# devParentRelType # devParentRelType
if ensure_column(self.sql, "Devices", "devParentRelType", "TEXT") is False: if ensure_column(self.sql, "Devices", "devParentRelType", "TEXT") is False:
return # addition failed return # addition failed
# devRequireNicsOnline # devRequireNicsOnline
if ensure_column(self.sql, "Devices", "devReqNicsOnline", "INTEGER") is False: if ensure_column(self.sql, "Devices", "devReqNicsOnline", "INTEGER") is False:
return # addition failed return # addition failed
# Settings table setup # Settings table setup
ensure_Settings(self.sql) ensure_Settings(self.sql)
# Parameters tables setup # Parameters tables setup
ensure_Parameters(self.sql) ensure_Parameters(self.sql)
# Plugins tables setup # Plugins tables setup
ensure_plugins_tables(self.sql) ensure_plugins_tables(self.sql)
# CurrentScan table setup # CurrentScan table setup
ensure_CurrentScan(self.sql) ensure_CurrentScan(self.sql)
# Views # Views
ensure_views(self.sql) ensure_views(self.sql)
# Views # Views
ensure_Indexes(self.sql) ensure_Indexes(self.sql)
# commit changes # commit changes
self.commitDB() self.commitDB()
# Init the AppEvent database table # Init the AppEvent database table
AppEvent_obj(self) AppEvent_obj(self)
@@ -150,7 +154,7 @@ class DB():
try: try:
result = get_table_json(self.sql, sqlQuery) result = get_table_json(self.sql, sqlQuery)
except Exception as e: except Exception as e:
mylog('verbose', ['[Database] - get_table_as_json ERROR:', e]) mylog('minimal', ['[Database] - get_table_as_json ERROR:', e])
return json_obj({}, []) # return empty object on failure return json_obj({}, []) # return empty object on failure
# mylog('debug',[ '[Database] - get_table_as_json - returning ', len(rows), " rows with columns: ", columnNames]) # mylog('debug',[ '[Database] - get_table_as_json - returning ', len(rows), " rows with columns: ", columnNames])
@@ -171,13 +175,13 @@ class DB():
rows = self.sql.fetchall() rows = self.sql.fetchall()
return rows return rows
except AssertionError: except AssertionError:
mylog('verbose',[ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args]) mylog('minimal', [ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args])
except sqlite3.Error as e: except sqlite3.Error as e:
mylog('verbose',[ '[Database] - SQL ERROR: ', e]) mylog('minimal', [ '[Database] - SQL ERROR: ', e])
return None return None
def read_one(self, query, *args): def read_one(self, query, *args):
""" """
call read() with the same arguments but only returns the first row. call read() with the same arguments but only returns the first row.
should only be used when there is a single row result expected should only be used when there is a single row result expected
""" """
@@ -186,8 +190,8 @@ class DB():
rows = self.read(query, *args) rows = self.read(query, *args)
if len(rows) == 1: if len(rows) == 1:
return rows[0] return rows[0]
if len(rows) > 1: if len(rows) > 1:
mylog('verbose',[ '[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args]) mylog('verbose',[ '[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args])
return rows[0] return rows[0]
# empty result set # empty result set
@@ -199,19 +203,22 @@ class DB():
def get_device_stats(db): def get_device_stats(db):
# columns = ["online","down","all","archived","new","unknown"] # columns = ["online","down","all","archived","new","unknown"]
return db.read_one(sql_devices_stats) return db.read_one(sql_devices_stats)
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
def get_all_devices(db): def get_all_devices(db):
return db.read(sql_devices_all) return db.read(sql_devices_all)
#------------------------------------------------------------------------------- #-------------------------------------------------------------------------------
def get_array_from_sql_rows(rows): def get_array_from_sql_rows(rows):
# Convert result into list of lists # Convert result into list of lists
arr = [] arr = []
for row in rows: for row in rows:
if isinstance(row, sqlite3.Row): if isinstance(row, sqlite3.Row):
arr.append(list(row)) # Convert row to list arr.append(list(row)) # Convert row to list
elif isinstance(row, (tuple, list)): elif isinstance(row, (tuple, list)):
arr.append(list(row)) # Already iterable, just convert to list arr.append(list(row)) # Already iterable, just convert to list
else: else:
arr.append([row]) # Handle single values safely arr.append([row]) # Handle single values safely