vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2025-12-07 09:36:05 -08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Removal of DHCP_Leases and PiHole DB tables

Browse Source
This commit is contained in:
Jokob-sk
2023-08-26 12:31:49 +10:00
parent ed0276b61e
commit b114de1fd9
26 changed files with 250 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pialert/README.md
View File

@@ -42,9 +42,6 @@ different methods to scan the network for devices or to find more details about
| Module | Description |
|--------|-----------|
|```__init__.py```| an empty init file (oops missing in the repo)|
|```arpscan.py```| run an arp-scan to discover devices |
|```internet.py```| discover the internet interface and check the external IP also manage Dynamic DNS |
|```nmapscan.py```| use Nmap to discover more about devices |
|```pholusscan.py```| use a 3rd party script Pholus to detect more details about the devices on the network |
|```pihole.py```| Use the PiHole network table in its db and also read the DHCP leases file to discover new devices |

23
pialert/database.py
View File

@@ -432,10 +432,31 @@ class DB():
cur_MAC STRING(50) NOT NULL COLLATE NOCASE,
cur_IP STRING(50) NOT NULL COLLATE NOCASE,
cur_Vendor STRING(250),
cur_ScanMethod STRING(10)
cur_ScanMethod STRING(10),
cur_Name STRING(250),
cur_LastQuery STRING(250),
cur_DateTime STRING(250)
);
""")
# indicates, if DHCP_Leases table is available
DHCP_LeasesMissing = self.sql.execute("""
SELECT name FROM sqlite_master WHERE type='table'
AND name='DHCP_Leases';
""").fetchone() == None
if DHCP_LeasesMissing == False:
self.sql.execute("DROP TABLE DHCP_Leases;")
# indicates, if PiHole_Network table is available
PiHole_NetworkMissing = self.sql.execute("""
SELECT name FROM sqlite_master WHERE type='table'
AND name='PiHole_Network';
""").fetchone() == None
if PiHole_NetworkMissing == False:
self.sql.execute("DROP TABLE PiHole_Network;")
self.commitDB()

243
pialert/device.py
View File

@@ -71,63 +71,39 @@ def save_scanned_devices (db):
sql.execute (f"""INSERT INTO CurrentScan (cur_ScanCycle, cur_MAC, cur_IP, cur_Vendor, cur_ScanMethod) VALUES ( 1, '{local_mac}', '{local_ip}', Null, 'local_MAC') """)
#-------------------------------------------------------------------------------
def print_scan_stats (db):
sql = db.sql #TO-DO
# Devices Detected
sql.execute ("""SELECT COUNT(*) FROM CurrentScan""")
mylog('verbose', ['[Scan Stats] Devices Detected.......: ', str (sql.fetchone()[0]) ])
def print_scan_stats(db):
sql = db.sql # TO-DO
# Devices arp-scan
sql.execute ("""SELECT COUNT(*) FROM CurrentScan WHERE cur_ScanMethod='arp-scan' """)
mylog('verbose', ['[Scan Stats] arp-scan detected..: ', str (sql.fetchone()[0]) ])
query = """
SELECT
(SELECT COUNT(*) FROM CurrentScan) AS devices_detected,
(SELECT COUNT(*) FROM CurrentScan WHERE NOT EXISTS (SELECT 1 FROM Devices WHERE dev_MAC = cur_MAC)) AS new_devices,
(SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS down_alerts,
(SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown = 1 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS new_down_alerts,
(SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 0) AS new_connections,
(SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS disconnections,
(SELECT COUNT(*) FROM Devices, CurrentScan WHERE dev_MAC = cur_MAC AND dev_LastIP <> cur_IP) AS ip_changes,
cur_ScanMethod,
COUNT(*) AS scan_method_count
FROM CurrentScan
GROUP BY cur_ScanMethod
"""
# Devices Pi-hole
sql.execute ("""SELECT COUNT(*) FROM CurrentScan WHERE cur_ScanMethod='PiHole'""")
mylog('verbose', ['[Scan Stats] Pi-hole detected...: +' + str (sql.fetchone()[0]) ])
sql.execute(query)
stats = sql.fetchall()
# New Devices
sql.execute ("""SELECT COUNT(*) FROM CurrentScan
WHERE NOT EXISTS (SELECT 1 FROM Devices
WHERE dev_MAC = cur_MAC) """)
mylog('verbose', ['[Scan Stats] New Devices........: ' + str (sql.fetchone()[0]) ])
# Down Alerts
sql.execute ("""SELECT COUNT(*) FROM Devices
WHERE dev_AlertDeviceDown = 1
AND NOT EXISTS (SELECT 1 FROM CurrentScan
WHERE dev_MAC = cur_MAC
) """)
mylog('verbose', ['[Scan Stats] Down Alerts........: ' + str (sql.fetchone()[0]) ])
# New Down Alerts
sql.execute ("""SELECT COUNT(*) FROM Devices
WHERE dev_AlertDeviceDown = 1
AND dev_PresentLastScan = 1
AND NOT EXISTS (SELECT 1 FROM CurrentScan
WHERE dev_MAC = cur_MAC
) """)
mylog('verbose', ['[Scan Stats] New Down Alerts....: ' + str (sql.fetchone()[0]) ])
# New Connections
sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan
WHERE dev_MAC = cur_MAC
AND dev_PresentLastScan = 0""")
mylog('verbose', ['[Scan Stats] New Connections....: ' + str ( sql.fetchone()[0]) ])
# Disconnections
sql.execute ("""SELECT COUNT(*) FROM Devices
WHERE dev_PresentLastScan = 1
AND NOT EXISTS (SELECT 1 FROM CurrentScan
WHERE dev_MAC = cur_MAC
) """)
mylog('verbose', ['[Scan Stats] Disconnections.....: ' + str ( sql.fetchone()[0]) ])
# IP Changes
sql.execute ("""SELECT COUNT(*) FROM Devices, CurrentScan
WHERE dev_MAC = cur_MAC
AND dev_LastIP <> cur_IP """)
mylog('verbose', ['[Scan Stats] IP Changes.........: ' + str ( sql.fetchone()[0]) ])
mylog('verbose', f'[Scan Stats] Devices Detected.......: {stats[0]["devices_detected"]}')
mylog('verbose', f'[Scan Stats] New Devices..........: {stats[0]["new_devices"]}')
mylog('verbose', f'[Scan Stats] Down Alerts..........: {stats[0]["down_alerts"]}')
mylog('verbose', f'[Scan Stats] New Down Alerts......: {stats[0]["new_down_alerts"]}')
mylog('verbose', f'[Scan Stats] New Connections......: {stats[0]["new_connections"]}')
mylog('verbose', f'[Scan Stats] Disconnections.......: {stats[0]["disconnections"]}')
mylog('verbose', f'[Scan Stats] IP Changes...........: {stats[0]["ip_changes"]}')
mylog('verbose', '[Scan Stats] Scan Method Statistics:')
for row in stats:
if row["cur_ScanMethod"] is not None:
mylog('verbose', f' {row["cur_ScanMethod"]}: {row["scan_method_count"]}')
#-------------------------------------------------------------------------------
@@ -135,7 +111,7 @@ def create_new_devices (db):
sql = db.sql # TO-DO
startTime = timeNowTZ()
# arpscan - Insert events for new devices
# Insert events for new devices from CurrentScan
mylog('debug','[New Devices] New devices - 1 Events')
sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
eve_EventType, eve_AdditionalInfo,
@@ -153,7 +129,7 @@ def create_new_devices (db):
WHERE NOT EXISTS (SELECT 1 FROM Sessions
WHERE ses_MAC = cur_MAC) """)
# arpscan - Create new devices
# Create new devices from CurrentScan
mylog('debug','[New Devices] 2 Create devices')
# default New Device values preparation
@@ -192,80 +168,82 @@ def create_new_devices (db):
'{get_setting_value('NEWDEV_dev_Icon')}'
"""
sqlQuery = f"""INSERT INTO Devices (dev_MAC, dev_name, dev_Vendor,
sqlQuery = f"""INSERT OR IGNORE INTO Devices (dev_MAC, dev_name, dev_Vendor,
dev_LastIP, dev_FirstConnection, dev_LastConnection,
{newDevColumns})
SELECT cur_MAC, '(unknown)', cur_Vendor, cur_IP, ?, ?,
SELECT cur_MAC,
CASE WHEN LENGTH(TRIM(cur_Name)) > 0 THEN cur_Name
ELSE '(unknown)' END,
cur_Vendor, cur_IP, ?, ?,
{newDevDefaults}
FROM CurrentScan
WHERE NOT EXISTS (SELECT 1 FROM Devices
WHERE dev_MAC = cur_MAC) """
FROM CurrentScan"""
# mylog('debug',f'[New Devices] 2 Create devices SQL: {sqlQuery}')
sql.execute (sqlQuery, (startTime, startTime) )
# Pi-hole - Insert events for new devices
# NOT STRICYLY NECESARY (Devices can be created through Current_Scan)
# NOT STRICYLY NECESARY (Devices can be created through CurrentScan)
# Bugfix #2 - Pi-hole devices w/o IP
mylog('debug','[New Devices] 3 Pi-hole Events')
sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
eve_EventType, eve_AdditionalInfo,
eve_PendingAlertEmail)
SELECT PH_MAC, IFNULL (PH_IP,'-'), ?, 'New Device',
'(Pi-Hole) ' || PH_Vendor, 1
FROM PiHole_Network
WHERE NOT EXISTS (SELECT 1 FROM Devices
WHERE dev_MAC = PH_MAC) """,
(startTime, ) )
# mylog('debug','[New Devices] 3 Pi-hole Events')
# sql.execute ("""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
# eve_EventType, eve_AdditionalInfo,
# eve_PendingAlertEmail)
# SELECT PH_MAC, IFNULL (PH_IP,'-'), ?, 'New Device',
# '(Pi-Hole) ' || PH_Vendor, 1
# FROM PiHole_Network
# WHERE NOT EXISTS (SELECT 1 FROM Devices
# WHERE dev_MAC = PH_MAC) """,
# (startTime, ) )
# Pi-hole - Create New Devices
# Bugfix #2 - Pi-hole devices w/o IP
mylog('debug','[New Devices] 4 Pi-hole Create devices')
# # Pi-hole - Create New Devices
# # Bugfix #2 - Pi-hole devices w/o IP
# mylog('debug','[New Devices] 4 Pi-hole Create devices')
sqlQuery = f"""INSERT INTO Devices (dev_MAC, dev_name, dev_Vendor,
dev_LastIP, dev_FirstConnection, dev_LastConnection,
{newDevColumns})
SELECT PH_MAC, PH_Name, PH_Vendor, IFNULL (PH_IP,'-'),
?, ?,
{newDevDefaults}
FROM PiHole_Network
WHERE NOT EXISTS (SELECT 1 FROM Devices
WHERE dev_MAC = PH_MAC) """
# sqlQuery = f"""INSERT INTO Devices (dev_MAC, dev_name, dev_Vendor,
# dev_LastIP, dev_FirstConnection, dev_LastConnection,
# {newDevColumns})
# SELECT PH_MAC, PH_Name, PH_Vendor, IFNULL (PH_IP,'-'),
# ?, ?,
# {newDevDefaults}
# FROM PiHole_Network
# WHERE NOT EXISTS (SELECT 1 FROM Devices
# WHERE dev_MAC = PH_MAC) """
# mylog('debug',f'[New Devices] 4 Create devices SQL: {sqlQuery}')
sql.execute (sqlQuery, (startTime, startTime) )
# DHCP Leases - Insert events for new devices
mylog('debug','[New Devices] 5 DHCP Leases Events')
# # DHCP Leases - Insert events for new devices
# mylog('debug','[New Devices] 5 DHCP Leases Events')
sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
eve_EventType, eve_AdditionalInfo,
eve_PendingAlertEmail)
SELECT DHCP_MAC, DHCP_IP, '{startTime}', 'New Device', '(DHCP lease)',1
FROM DHCP_Leases
WHERE NOT EXISTS (SELECT 1 FROM Devices
WHERE dev_MAC = DHCP_MAC) """)
# sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
# eve_EventType, eve_AdditionalInfo,
# eve_PendingAlertEmail)
# SELECT DHCP_MAC, DHCP_IP, '{startTime}', 'New Device', '(DHCP lease)',1
# FROM DHCP_Leases
# WHERE NOT EXISTS (SELECT 1 FROM Devices
# WHERE dev_MAC = DHCP_MAC) """)
# DHCP Leases - Create New Devices
mylog('debug','[New Devices] 6 DHCP Leases Create devices')
# # DHCP Leases - Create New Devices
# mylog('debug','[New Devices] 6 DHCP Leases Create devices')
sqlQuery = f"""INSERT INTO Devices (dev_MAC, dev_name, dev_LastIP,
dev_Vendor, dev_FirstConnection, dev_LastConnection,
{newDevColumns})
SELECT DISTINCT DHCP_MAC,
(SELECT DHCP_Name FROM DHCP_Leases AS D2
WHERE D2.DHCP_MAC = D1.DHCP_MAC
ORDER BY DHCP_DateTime DESC LIMIT 1),
(SELECT DHCP_IP FROM DHCP_Leases AS D2
WHERE D2.DHCP_MAC = D1.DHCP_MAC
ORDER BY DHCP_DateTime DESC LIMIT 1),
'(unknown)', ?, ?,
{newDevDefaults}
FROM DHCP_Leases AS D1
WHERE NOT EXISTS (SELECT 1 FROM Devices
WHERE dev_MAC = DHCP_MAC) """
# sqlQuery = f"""INSERT INTO Devices (dev_MAC, dev_name, dev_LastIP,
# dev_Vendor, dev_FirstConnection, dev_LastConnection,
# {newDevColumns})
# SELECT DISTINCT DHCP_MAC,
# (SELECT DHCP_Name FROM DHCP_Leases AS D2
# WHERE D2.DHCP_MAC = D1.DHCP_MAC
# ORDER BY DHCP_DateTime DESC LIMIT 1),
# (SELECT DHCP_IP FROM DHCP_Leases AS D2
# WHERE D2.DHCP_MAC = D1.DHCP_MAC
# ORDER BY DHCP_DateTime DESC LIMIT 1),
# '(unknown)', ?, ?,
# {newDevDefaults}
# FROM DHCP_Leases AS D1
# WHERE NOT EXISTS (SELECT 1 FROM Devices
# WHERE dev_MAC = DHCP_MAC) """
# mylog('debug',f'[New Devices] 6 Create devices SQL: {sqlQuery}')
@@ -279,54 +257,44 @@ def create_new_devices (db):
def update_devices_data_from_scan (db):
sql = db.sql #TO-DO
startTime = timeNowTZ()
# Update Last Connection
mylog('debug','[Update Devices] 1 Last Connection')
sql.execute (f"""UPDATE Devices SET dev_LastConnection = '{startTime}',
mylog('debug', '[Update Devices] 1 Last Connection')
sql.execute(f"""UPDATE Devices SET dev_LastConnection = '{startTime}',
dev_PresentLastScan = 1
WHERE dev_PresentLastScan = 0
AND EXISTS (SELECT 1 FROM CurrentScan
WHERE dev_MAC = cur_MAC) """)
# Clean no active devices
mylog('debug','[Update Devices] 2 Clean no active devices')
sql.execute ("""UPDATE Devices SET dev_PresentLastScan = 0
mylog('debug', '[Update Devices] 2 Clean no active devices')
sql.execute("""UPDATE Devices SET dev_PresentLastScan = 0
WHERE NOT EXISTS (SELECT 1 FROM CurrentScan
WHERE dev_MAC = cur_MAC) """)
# Update IP & Vendor
mylog('debug','[Update Devices] - 3 LastIP & Vendor')
sql.execute ("""UPDATE Devices
mylog('debug', '[Update Devices] - 3 LastIP & Vendor')
sql.execute("""UPDATE Devices
SET dev_LastIP = (SELECT cur_IP FROM CurrentScan
WHERE dev_MAC = cur_MAC),
dev_Vendor = (SELECT cur_Vendor FROM CurrentScan
WHERE dev_MAC = cur_MAC
)
WHERE EXISTS (SELECT 1 FROM CurrentScan
WHERE dev_MAC = cur_MAC) """)
WHERE dev_MAC = cur_MAC) """)
# Pi-hole Network - Update (unknown) Name
# Update (unknown) or (name not found) Names if available
mylog('debug','[Update Devices] - 4 Unknown Name')
sql.execute ("""UPDATE Devices
SET dev_NAME = (SELECT PH_Name FROM PiHole_Network
WHERE PH_MAC = dev_MAC)
SET dev_NAME = (SELECT cur_Name FROM CurrentScan
WHERE cur_MAC = dev_MAC)
WHERE (dev_Name in ("(unknown)", "(name not found)", "" )
OR dev_Name IS NULL)
AND EXISTS (SELECT 1 FROM PiHole_Network
WHERE PH_MAC = dev_MAC
AND PH_NAME IS NOT NULL
AND PH_NAME <> '') """)
# DHCP Leases - Update (unknown) Name
sql.execute ("""UPDATE Devices
SET dev_NAME = (SELECT DHCP_Name FROM DHCP_Leases
WHERE DHCP_MAC = dev_MAC)
WHERE (dev_Name in ("(unknown)", "(name not found)", "" )
OR dev_Name IS NULL)
AND EXISTS (SELECT 1 FROM DHCP_Leases
WHERE DHCP_MAC = dev_MAC)""")
# DHCP Leases - Vendor
mylog('debug','[Update Devices] - 5 Vendor')
AND EXISTS (SELECT 1 FROM CurrentScan
WHERE cur_MAC = dev_MAC
AND cur_Name IS NOT NULL
AND cur_Name IS NOT 'null'
AND cur_Name <> '') """)
recordsToUpdate = []
query = """SELECT * FROM Devices
@@ -340,9 +308,8 @@ def update_devices_data_from_scan (db):
sql.executemany ("UPDATE Devices SET dev_Vendor = ? WHERE dev_MAC = ? ",
recordsToUpdate )
# clean-up device leases table
sql.execute ("DELETE FROM DHCP_Leases")
mylog('debug','[Update Devices] Update devices end')
#-------------------------------------------------------------------------------

22
pialert/plugin.py
View File

@@ -259,9 +259,9 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
# Create a tuple containing values to be inserted into the database.
# Each value corresponds to a column in the table in the order of the columns.
# must match the Plugins_Objects and Plugins_Events databse tables and can be used as input for the plugin_object_class
sqlParams.append(
sqlParams.append((
0, # "Index" placeholder
(plugin["unique_prefix"], # "Plugin"
plugin["unique_prefix"], # "Plugin"
row[0], # "Object_PrimaryID"
handle_empty(row[1]), # "Object_SecondaryID"
'null', # "DateTimeCreated" column (null placeholder)
@@ -536,8 +536,8 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr):
mylog('none', ['[Plugins] Error: ', e])
raise e
# Perform database table mapping if enabled for the plugin
if len(pluginEvents) > 0 and "mapped_to_table" in plugin:
# Perform database table mapping if enabled for the plugin
if len(pluginEvents) > 0 and "mapped_to_table" in plugin:
# Initialize an empty list to store SQL parameters.
sqlParams = []
@@ -558,6 +558,7 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr):
for clmn in plugin['database_column_definitions']:
if 'mapped_to_column' in clmn:
mappedCols.append(clmn)
columnsStr = f'{columnsStr}, "{clmn["mapped_to_column"]}"'
valuesStr = f'{valuesStr}, ?'
@@ -598,6 +599,10 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr):
elif col['column'] == 'Status':
tmpList.append(plgEv.status)
# Check if there's a default value specified for this column in the JSON.
if 'mapped_to_column_data' in col and 'value' in col['mapped_to_column_data']:
tmpList.append(col['mapped_to_column_data']['value'])
# Append the mapped values to the list 'sqlParams' as a tuple.
sqlParams.append(tuple(tmpList))
@@ -606,6 +611,7 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr):
# Log a debug message showing the generated SQL query for mapping.
mylog('debug', ['[Plugins] SQL query for mapping: ', q])
mylog('debug', ['[Plugins] SQL sqlParams for mapping: ', sqlParams])
# Execute the SQL query using 'sql.executemany()' and the 'sqlParams' list of tuples.
# This will insert multiple rows into the database in one go.
@@ -613,13 +619,13 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr):
db.commitDB()
# perform scan if mapped to CurrentScan table
if dbTable == 'CurrentScan':
pluginsState.processScan = True
# perform scan if mapped to CurrentScan table
if dbTable == 'CurrentScan':
pluginsState.processScan = True
db.commitDB()
return pluginsState