FQDN, Dig refactor, docs #1065

This commit is contained in:
jokob-sk
2025-06-01 13:59:54 +10:00
parent 941e838c74
commit f4a3717859
50 changed files with 941 additions and 874 deletions

View File

@@ -80,370 +80,10 @@ class DB():
"""
Check the current tables in the DB and upgrade them if neccessary
"""
self.sql.execute("""
CREATE TABLE IF NOT EXISTS "Online_History" (
"Index" INTEGER,
"Scan_Date" TEXT,
"Online_Devices" INTEGER,
"Down_Devices" INTEGER,
"All_Devices" INTEGER,
"Archived_Devices" INTEGER,
"Offline_Devices" INTEGER,
PRIMARY KEY("Index" AUTOINCREMENT)
);
""")
# -------------------------------------------------------------------
# DevicesNew - cleanup after 6/6/2025 - need to update also DB in the source code!
# check if migration already done based on devMac
devMac_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='devMac'
""").fetchone()[0] == 0
if devMac_missing:
# -------------------------------------------------------------------------
# Alter Devices table
# -------------------------------------------------------------------------
# dev_Network_Node_MAC_ADDR column
dev_Network_Node_MAC_ADDR_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_MAC_ADDR'
""").fetchone()[0] == 0
if dev_Network_Node_MAC_ADDR_missing :
mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_MAC_ADDR to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_Network_Node_MAC_ADDR" TEXT
""")
# dev_Network_Node_port column
dev_Network_Node_port_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_port'
""").fetchone()[0] == 0
if dev_Network_Node_port_missing :
mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_port to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_Network_Node_port" INTEGER
""")
# dev_Icon column
dev_Icon_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Icon'
""").fetchone()[0] == 0
if dev_Icon_missing :
mylog('verbose', ["[upgradeDB] Adding dev_Icon to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_Icon" TEXT
""")
# dev_GUID column
dev_GUID_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_GUID'
""").fetchone()[0] == 0
if dev_GUID_missing :
mylog('verbose', ["[upgradeDB] Adding dev_GUID to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_GUID" TEXT
""")
# dev_NetworkSite column
dev_NetworkSite_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_NetworkSite'
""").fetchone()[0] == 0
if dev_NetworkSite_missing :
mylog('verbose', ["[upgradeDB] Adding dev_NetworkSite to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_NetworkSite" TEXT
""")
# dev_SSID column
dev_SSID_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_SSID'
""").fetchone()[0] == 0
if dev_SSID_missing :
mylog('verbose', ["[upgradeDB] Adding dev_SSID to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_SSID" TEXT
""")
# SQL query to update missing dev_GUID
self.sql.execute(f'''
UPDATE Devices
SET dev_GUID = {sql_generateGuid}
WHERE dev_GUID IS NULL
''')
# dev_SyncHubNodeName column
dev_SyncHubNodeName_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_SyncHubNodeName'
""").fetchone()[0] == 0
if dev_SyncHubNodeName_missing :
mylog('verbose', ["[upgradeDB] Adding dev_SyncHubNodeName to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_SyncHubNodeName" TEXT
""")
# dev_SourcePlugin column
dev_SourcePlugin_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_SourcePlugin'
""").fetchone()[0] == 0
if dev_SourcePlugin_missing :
mylog('verbose', ["[upgradeDB] Adding dev_SourcePlugin to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "dev_SourcePlugin" TEXT
""")
# SQL to create Devices table with indexes
sql_create_devices_new_tmp = """
CREATE TABLE IF NOT EXISTS Devices_tmp (
devMac STRING (50) PRIMARY KEY NOT NULL COLLATE NOCASE,
devName STRING (50) NOT NULL DEFAULT "(unknown)",
devOwner STRING (30) DEFAULT "(unknown)" NOT NULL,
devType STRING (30),
devVendor STRING (250),
devFavorite BOOLEAN CHECK (devFavorite IN (0, 1)) DEFAULT (0) NOT NULL,
devGroup STRING (10),
devComments TEXT,
devFirstConnection DATETIME NOT NULL,
devLastConnection DATETIME NOT NULL,
devLastIP STRING (50) NOT NULL COLLATE NOCASE,
devStaticIP BOOLEAN DEFAULT (0) NOT NULL CHECK (devStaticIP IN (0, 1)),
devScan INTEGER DEFAULT (1) NOT NULL,
devLogEvents BOOLEAN NOT NULL DEFAULT (1) CHECK (devLogEvents IN (0, 1)),
devAlertEvents BOOLEAN NOT NULL DEFAULT (1) CHECK (devAlertEvents IN (0, 1)),
devAlertDown BOOLEAN NOT NULL DEFAULT (0) CHECK (devAlertDown IN (0, 1)),
devSkipRepeated INTEGER DEFAULT 0 NOT NULL,
devLastNotification DATETIME,
devPresentLastScan BOOLEAN NOT NULL DEFAULT (0) CHECK (devPresentLastScan IN (0, 1)),
devIsNew BOOLEAN NOT NULL DEFAULT (1) CHECK (devIsNew IN (0, 1)),
devLocation STRING (250) COLLATE NOCASE,
devIsArchived BOOLEAN NOT NULL DEFAULT (0) CHECK (devIsArchived IN (0, 1)),
devParentMAC TEXT,
devParentPort INTEGER,
devIcon TEXT,
devGUID TEXT,
devSite TEXT,
devSSID TEXT,
devSyncHubNode TEXT,
devSourcePlugin TEXT
);
CREATE INDEX IF NOT EXISTS IDX_dev_PresentLastScan ON Devices_tmp (devPresentLastScan);
CREATE INDEX IF NOT EXISTS IDX_dev_FirstConnection ON Devices_tmp (devFirstConnection);
CREATE INDEX IF NOT EXISTS IDX_dev_AlertDeviceDown ON Devices_tmp (devAlertDown);
CREATE INDEX IF NOT EXISTS IDX_dev_StaticIP ON Devices_tmp (devStaticIP);
CREATE INDEX IF NOT EXISTS IDX_dev_ScanCycle ON Devices_tmp (devScan);
CREATE INDEX IF NOT EXISTS IDX_dev_Favorite ON Devices_tmp (devFavorite);
CREATE INDEX IF NOT EXISTS IDX_dev_LastIP ON Devices_tmp (devLastIP);
CREATE INDEX IF NOT EXISTS IDX_dev_NewDevice ON Devices_tmp (devIsNew);
CREATE INDEX IF NOT EXISTS IDX_dev_Archived ON Devices_tmp (devIsArchived);
"""
# Execute the creation of the Devices table and indexes
self.sql.executescript(sql_create_devices_new_tmp)
# copy over data
sql_copy_from_devices = """
INSERT OR IGNORE INTO Devices_tmp (
devMac,
devName,
devOwner,
devType,
devVendor,
devFavorite,
devGroup,
devComments,
devFirstConnection,
devLastConnection,
devLastIP,
devStaticIP,
devScan,
devLogEvents,
devAlertEvents,
devAlertDown,
devSkipRepeated,
devLastNotification,
devPresentLastScan,
devIsNew,
devLocation,
devIsArchived,
devParentMAC,
devParentPort,
devIcon,
devGUID,
devSite,
devSSID,
devSyncHubNode,
devSourcePlugin
)
SELECT
dev_MAC AS devMac,
dev_Name AS devName,
dev_Owner AS devOwner,
dev_DeviceType AS devType,
dev_Vendor AS devVendor,
dev_Favorite AS devFavorite,
dev_Group AS devGroup,
dev_Comments AS devComments,
dev_FirstConnection AS devFirstConnection,
dev_LastConnection AS devLastConnection,
dev_LastIP AS devLastIP,
dev_StaticIP AS devStaticIP,
dev_ScanCycle AS devScan,
dev_LogEvents AS devLogEvents,
dev_AlertEvents AS devAlertEvents,
dev_AlertDeviceDown AS devAlertDown,
dev_SkipRepeated AS devSkipRepeated,
dev_LastNotification AS devLastNotification,
dev_PresentLastScan AS devPresentLastScan,
dev_NewDevice AS devIsNew,
dev_Location AS devLocation,
dev_Archived AS devIsArchived,
dev_Network_Node_MAC_ADDR AS devParentMAC,
dev_Network_Node_port AS devParentPort,
dev_Icon AS devIcon,
dev_GUID AS devGUID,
dev_NetworkSite AS devSite,
dev_SSID AS devSSID,
dev_SyncHubNodeName AS devSyncHubNode,
dev_SourcePlugin AS devSourcePlugin
FROM Devices;
"""
self.sql.execute(sql_copy_from_devices)
self.sql.execute(""" DROP TABLE Devices;""")
# SQL to create Devices table with indexes
sql_create_devices_new = """
CREATE TABLE IF NOT EXISTS Devices (
devMac STRING (50) PRIMARY KEY NOT NULL COLLATE NOCASE,
devName STRING (50) NOT NULL DEFAULT "(unknown)",
devOwner STRING (30) DEFAULT "(unknown)" NOT NULL,
devType STRING (30),
devVendor STRING (250),
devFavorite BOOLEAN CHECK (devFavorite IN (0, 1)) DEFAULT (0) NOT NULL,
devGroup STRING (10),
devComments TEXT,
devFirstConnection DATETIME NOT NULL,
devLastConnection DATETIME NOT NULL,
devLastIP STRING (50) NOT NULL COLLATE NOCASE,
devStaticIP BOOLEAN DEFAULT (0) NOT NULL CHECK (devStaticIP IN (0, 1)),
devScan INTEGER DEFAULT (1) NOT NULL,
devLogEvents BOOLEAN NOT NULL DEFAULT (1) CHECK (devLogEvents IN (0, 1)),
devAlertEvents BOOLEAN NOT NULL DEFAULT (1) CHECK (devAlertEvents IN (0, 1)),
devAlertDown BOOLEAN NOT NULL DEFAULT (0) CHECK (devAlertDown IN (0, 1)),
devSkipRepeated INTEGER DEFAULT 0 NOT NULL,
devLastNotification DATETIME,
devPresentLastScan BOOLEAN NOT NULL DEFAULT (0) CHECK (devPresentLastScan IN (0, 1)),
devIsNew BOOLEAN NOT NULL DEFAULT (1) CHECK (devIsNew IN (0, 1)),
devLocation STRING (250) COLLATE NOCASE,
devIsArchived BOOLEAN NOT NULL DEFAULT (0) CHECK (devIsArchived IN (0, 1)),
devParentMAC TEXT,
devParentPort INTEGER,
devIcon TEXT,
devGUID TEXT,
devSite TEXT,
devSSID TEXT,
devSyncHubNode TEXT,
devSourcePlugin TEXT
);
CREATE INDEX IF NOT EXISTS IDX_dev_PresentLastScan ON Devices (devPresentLastScan);
CREATE INDEX IF NOT EXISTS IDX_dev_FirstConnection ON Devices (devFirstConnection);
CREATE INDEX IF NOT EXISTS IDX_dev_AlertDeviceDown ON Devices (devAlertDown);
CREATE INDEX IF NOT EXISTS IDX_dev_StaticIP ON Devices (devStaticIP);
CREATE INDEX IF NOT EXISTS IDX_dev_ScanCycle ON Devices (devScan);
CREATE INDEX IF NOT EXISTS IDX_dev_Favorite ON Devices (devFavorite);
CREATE INDEX IF NOT EXISTS IDX_dev_LastIP ON Devices (devLastIP);
CREATE INDEX IF NOT EXISTS IDX_dev_NewDevice ON Devices (devIsNew);
CREATE INDEX IF NOT EXISTS IDX_dev_Archived ON Devices (devIsArchived);
"""
# Execute the creation of the Devices table and indexes
self.sql.executescript(sql_create_devices_new)
# copy over data
sql_copy_from_devices_tmp = """
INSERT OR IGNORE INTO Devices (
devMac,
devName,
devOwner,
devType,
devVendor,
devFavorite,
devGroup,
devComments,
devFirstConnection,
devLastConnection,
devLastIP,
devStaticIP,
devScan,
devLogEvents,
devAlertEvents,
devAlertDown,
devSkipRepeated,
devLastNotification,
devPresentLastScan,
devIsNew,
devLocation,
devIsArchived,
devParentMAC,
devParentPort,
devIcon,
devGUID,
devSite,
devSSID,
devSyncHubNode,
devSourcePlugin
)
SELECT
devMac,
devName,
devOwner,
devType,
devVendor,
devFavorite,
devGroup,
devComments,
devFirstConnection,
devLastConnection,
devLastIP,
devStaticIP,
devScan,
devLogEvents,
devAlertEvents,
devAlertDown,
devSkipRepeated,
devLastNotification,
devPresentLastScan,
devIsNew,
devLocation,
devIsArchived,
devParentMAC,
devParentPort,
devIcon,
devGUID,
devSite,
devSSID,
devSyncHubNode,
devSourcePlugin
FROM Devices_tmp;
"""
self.sql.execute(sql_copy_from_devices_tmp)
self.sql.execute(""" DROP TABLE Devices_tmp;""")
# -------------------------------------------------------------------------
# Alter Devices table
# -------------------------------------------------------------------------
# VIEWS
@@ -477,17 +117,19 @@ class DB():
# add fields if missing
# devCustomProps column
devCustomProps_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='devCustomProps'
# devFQDN missing?
devFQDN_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='devFQDN'
""").fetchone()[0] == 0
if devFQDN_missing:
if devCustomProps_missing :
mylog('verbose', ["[upgradeDB] Adding devCustomProps to the Devices table"])
mylog('verbose', ["[upgradeDB] Adding devFQDN to the Devices table"])
self.sql.execute("""
ALTER TABLE "Devices" ADD "devCustomProps" TEXT
ALTER TABLE "Devices" ADD "devFQDN" TEXT
""")
# -------------------------------------------------------------------------
# Settings table setup
@@ -564,48 +206,6 @@ class DB():
); """
self.sql.execute(sql_Plugins_Objects)
# -----------------------------------------
# REMOVE after 6/6/2025 - START
# -----------------------------------------
# syncHubNodeName column
plug_SyncHubNodeName_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_Objects') WHERE name='SyncHubNodeName'
""").fetchone()[0] == 0
if plug_SyncHubNodeName_missing :
mylog('verbose', ["[upgradeDB] Adding SyncHubNodeName to the Plugins_Objects table"])
self.sql.execute("""
ALTER TABLE "Plugins_Objects" ADD "SyncHubNodeName" TEXT
""")
# helper columns HelpVal1-4
plug_HelpValues_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_Objects') WHERE name='HelpVal1'
""").fetchone()[0] == 0
if plug_HelpValues_missing :
mylog('verbose', ["[upgradeDB] Adding HelpVal1-4 to the Plugins_Objects table"])
self.sql.execute('ALTER TABLE "Plugins_Objects" ADD COLUMN "HelpVal1" TEXT')
self.sql.execute('ALTER TABLE "Plugins_Objects" ADD COLUMN "HelpVal2" TEXT')
self.sql.execute('ALTER TABLE "Plugins_Objects" ADD COLUMN "HelpVal3" TEXT')
self.sql.execute('ALTER TABLE "Plugins_Objects" ADD COLUMN "HelpVal4" TEXT')
# plug_ObjectGUID_missing column
plug_ObjectGUID_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_Objects') WHERE name='ObjectGUID'
""").fetchone()[0] == 0
if plug_ObjectGUID_missing :
mylog('verbose', ["[upgradeDB] Adding ObjectGUID to the Plugins_Objects table"])
self.sql.execute("""
ALTER TABLE "Plugins_Objects" ADD "ObjectGUID" TEXT
""")
# -----------------------------------------
# REMOVE after 6/6/2025 - END
# -----------------------------------------
# Plugin execution results
sql_Plugins_Events = """ CREATE TABLE IF NOT EXISTS Plugins_Events(
"Index" INTEGER,
@@ -631,49 +231,6 @@ class DB():
); """
self.sql.execute(sql_Plugins_Events)
# -----------------------------------------
# REMOVE after 6/6/2025 - START
# -----------------------------------------
# syncHubNodeName column
plug_SyncHubNodeName_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_Events') WHERE name='SyncHubNodeName'
""").fetchone()[0] == 0
if plug_SyncHubNodeName_missing :
mylog('verbose', ["[upgradeDB] Adding SyncHubNodeName to the Plugins_Events table"])
self.sql.execute("""
ALTER TABLE "Plugins_Events" ADD "SyncHubNodeName" TEXT
""")
# helper columns HelpVal1-4
plug_HelpValues_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_Events') WHERE name='HelpVal1'
""").fetchone()[0] == 0
if plug_HelpValues_missing :
mylog('verbose', ["[upgradeDB] Adding HelpVal1-4 to the Plugins_Events table"])
self.sql.execute('ALTER TABLE "Plugins_Events" ADD COLUMN "HelpVal1" TEXT')
self.sql.execute('ALTER TABLE "Plugins_Events" ADD COLUMN "HelpVal2" TEXT')
self.sql.execute('ALTER TABLE "Plugins_Events" ADD COLUMN "HelpVal3" TEXT')
self.sql.execute('ALTER TABLE "Plugins_Events" ADD COLUMN "HelpVal4" TEXT')
# plug_ObjectGUID_missing column
plug_ObjectGUID_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_Events') WHERE name='ObjectGUID'
""").fetchone()[0] == 0
if plug_ObjectGUID_missing :
mylog('verbose', ["[upgradeDB] Adding ObjectGUID to the Plugins_Events table"])
self.sql.execute("""
ALTER TABLE "Plugins_Events" ADD "ObjectGUID" TEXT
""")
# -----------------------------------------
# REMOVE after 6/6/2025 - END
# -----------------------------------------
# Plugin execution history
sql_Plugins_History = """ CREATE TABLE IF NOT EXISTS Plugins_History(
"Index" INTEGER,
@@ -699,48 +256,6 @@ class DB():
); """
self.sql.execute(sql_Plugins_History)
# -----------------------------------------
# REMOVE after 6/6/2025 - START
# -----------------------------------------
# syncHubNodeName column
plug_SyncHubNodeName_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_History') WHERE name='SyncHubNodeName'
""").fetchone()[0] == 0
if plug_SyncHubNodeName_missing :
mylog('verbose', ["[upgradeDB] Adding SyncHubNodeName to the Plugins_History table"])
self.sql.execute("""
ALTER TABLE "Plugins_History" ADD "SyncHubNodeName" TEXT
""")
# helper columns HelpVal1-4
plug_HelpValues_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_History') WHERE name='HelpVal1'
""").fetchone()[0] == 0
if plug_HelpValues_missing :
mylog('verbose', ["[upgradeDB] Adding HelpVal1-4 to the Plugins_History table"])
self.sql.execute('ALTER TABLE "Plugins_History" ADD COLUMN "HelpVal1" TEXT')
self.sql.execute('ALTER TABLE "Plugins_History" ADD COLUMN "HelpVal2" TEXT')
self.sql.execute('ALTER TABLE "Plugins_History" ADD COLUMN "HelpVal3" TEXT')
self.sql.execute('ALTER TABLE "Plugins_History" ADD COLUMN "HelpVal4" TEXT')
# plug_ObjectGUID_missing column
plug_ObjectGUID_missing = self.sql.execute ("""
SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Plugins_History') WHERE name='ObjectGUID'
""").fetchone()[0] == 0
if plug_ObjectGUID_missing :
mylog('verbose', ["[upgradeDB] Adding ObjectGUID to the Plugins_History table"])
self.sql.execute("""
ALTER TABLE "Plugins_History" ADD "ObjectGUID" TEXT
""")
# -----------------------------------------
# REMOVE after 6/6/2025 - END
# -----------------------------------------
# -------------------------------------------------------------------------
# Plugins_Language_Strings table setup
@@ -845,21 +360,6 @@ class DB():
# Init the AppEvent database table
AppEvent_obj(self)
# -------------------------------------------------------------------------
# DELETING OBSOLETE TABLES - to remove with updated db file after 9/9/2024
# -------------------------------------------------------------------------
# Deletes obsolete ScanCycles
self.sql.execute(""" DROP TABLE IF EXISTS ScanCycles;""")
self.sql.execute(""" DROP TABLE IF EXISTS DHCP_Leases;""")
self.sql.execute(""" DROP TABLE IF EXISTS PiHole_Network;""")
self.commitDB()
# -------------------------------------------------------------------------
# DELETING OBSOLETE TABLES - to remove with updated db file after 9/9/2024
# -------------------------------------------------------------------------
#-------------------------------------------------------------------------------
def get_table_as_json(self, sqlQuery):