vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2026-04-11 12:41:36 -07:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

db_cleanup plugin

Browse Source
This commit is contained in:
Jokob-sk
2023-09-17 21:46:05 +10:00
parent 80de181827
commit a5b952f18c
13 changed files with 370 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
pialert/__main__.py
View File

@@ -53,16 +53,13 @@ main structure of Pi Alert
(re)import plugin config
run plugins (once)
run frontend events
update API
run scans
run plugins (scheduled)
check internet IP
check vendor
run "scan_network()"
processing scan results
run plugins (after Scan)
reporting
cleanup
update API
run plugins (scheduled)
check internet IP
check vendor
processing scan results
run plugins (after Scan)
reporting
end loop
"""
@@ -103,10 +100,8 @@ def main ():
# TODO fix these
loop_start_time = conf.loop_start_time # TODO fix
last_update_vendors = conf.last_update_vendors
last_cleanup = conf.last_cleanup
last_version_check = conf.last_version_check
last_update_vendors = conf.last_update_vendors
last_version_check = conf.last_version_check
# check if new version is available / only check once an hour
if conf.last_version_check + datetime.timedelta(hours=1) < conf.loop_start_time :
@@ -186,13 +181,6 @@ def main ():
# send all configured notifications
send_notifications(db)
# clean up the DB once an hour
if conf.last_cleanup + datetime.timedelta(hours = 1) < loop_start_time:
conf.last_cleanup = loop_start_time
conf.cycle = 'cleanup'
mylog('verbose', ['[MAIN] cycle:',conf.cycle])
db.cleanup_database(startTime, conf.DAYS_TO_KEEP_EVENTS, get_setting_value('PHOLUS_DAYS_DATA'), conf.HRS_TO_KEEP_NEWDEV, conf.PLUGINS_KEEP_HIST)
# Commit SQL
db.commitDB()

2
pialert/api.py
View File

@@ -75,7 +75,7 @@ class api_endpoint_class:
index = index + 1
# cehck if API endpoints have changed or if it's a new one
# check if API endpoints have changed or if it's a new one
if not found or changed:
mylog('verbose', [f'[API] Updating {self.fileName} file in /front/api'])

1
pialert/conf.py
View File

@@ -23,7 +23,6 @@ time_started = ''
startTime = ''
last_internet_IP_scan = ''
last_scan_run = ''
last_cleanup = ''
last_update_vendors = ''
last_version_check = ''
arpscan_devices = []

80
pialert/database.py
View File

@@ -74,86 +74,6 @@ class DB():
return arr
#===============================================================================
# Cleanup / upkeep database
#===============================================================================
def cleanup_database (self, startTime, DAYS_TO_KEEP_EVENTS, PHOLUS_DAYS_DATA, HRS_TO_KEEP_NEWDEV, PLUGINS_KEEP_HIST):
"""
Cleaning out old records from the tables that don't need to keep all data.
"""
# Header
#updateState(self,"Upkeep: Clean DB")
mylog('verbose', ['[DB Cleanup] Upkeep Database:' ])
# Cleanup Online History
mylog('verbose', ['[DB Cleanup] Online_History: Delete all but keep latest 150 entries'])
self.sql.execute ("""DELETE from Online_History where "Index" not in (
SELECT "Index" from Online_History
order by Scan_Date desc limit 150)""")
mylog('verbose', ['[DB Cleanup] Optimize Database'])
# Cleanup Events
mylog('verbose', [f'[DB Cleanup] Events: Delete all older than {str(DAYS_TO_KEEP_EVENTS)} days (DAYS_TO_KEEP_EVENTS setting)'])
self.sql.execute (f"""DELETE FROM Events
WHERE eve_DateTime <= date('now', '-{str(DAYS_TO_KEEP_EVENTS)} day')""")
# Trim Plugins_History entries to less than PLUGINS_KEEP_HIST setting per unique "Plugin" column entry
mylog('verbose', [f'[DB Cleanup] Plugins_History: Trim Plugins_History entries to less than {str(PLUGINS_KEEP_HIST)} per Plugin (PLUGINS_KEEP_HIST setting)'])
# Build the SQL query to delete entries that exceed the limit per unique "Plugin" column entry
delete_query = f"""DELETE FROM Plugins_History
WHERE "Index" NOT IN (
SELECT "Index"
FROM (
SELECT "Index",
ROW_NUMBER() OVER(PARTITION BY "Plugin" ORDER BY DateTimeChanged DESC) AS row_num
FROM Plugins_History
) AS ranked_objects
WHERE row_num <= {str(PLUGINS_KEEP_HIST)}
);"""
self.sql.execute(delete_query)
# Cleanup Pholus_Scan
if PHOLUS_DAYS_DATA != 0:
mylog('verbose', ['[DB Cleanup] Pholus_Scan: Delete all older than ' + str(PHOLUS_DAYS_DATA) + ' days (PHOLUS_DAYS_DATA setting)'])
# todo: improvement possibility: keep at least N per mac
self.sql.execute (f"""DELETE FROM Pholus_Scan
WHERE Time <= date('now', '-{str(PHOLUS_DAYS_DATA)} day')""")
# Cleanup New Devices
if HRS_TO_KEEP_NEWDEV != 0:
mylog('verbose', [f'[DB Cleanup] Devices: Delete all New Devices older than {str(HRS_TO_KEEP_NEWDEV)} hours (HRS_TO_KEEP_NEWDEV setting)'])
self.sql.execute (f"""DELETE FROM Devices
WHERE dev_NewDevice = 1 AND dev_FirstConnection < date('now', '+{str(HRS_TO_KEEP_NEWDEV)} hour')""")
# De-dupe (de-duplicate) from the Plugins_Objects table
# TODO This shouldn't be necessary - probably a concurrency bug somewhere in the code :(
mylog('verbose', ['[DB Cleanup] Plugins_Objects: Delete all duplicates'])
self.sql.execute("""
DELETE FROM Plugins_Objects
WHERE rowid > (
SELECT MIN(rowid) FROM Plugins_Objects p2
WHERE Plugins_Objects.Plugin = p2.Plugin
AND Plugins_Objects.Object_PrimaryID = p2.Object_PrimaryID
AND Plugins_Objects.Object_SecondaryID = p2.Object_SecondaryID
AND Plugins_Objects.UserData = p2.UserData
)
""")
# De-Dupe (de-duplicate - remove duplicate entries) from the Pholus_Scan table
mylog('verbose', ['[DB Cleanup] Pholus_Scan: Delete all duplicates'])
self.sql.execute ("""DELETE FROM Pholus_Scan
WHERE rowid > (
SELECT MIN(rowid) FROM Pholus_Scan p2
WHERE Pholus_Scan.MAC = p2.MAC
AND Pholus_Scan.Value = p2.Value
AND Pholus_Scan.Record_Type = p2.Record_Type
);""")
# Shrink DB
mylog('verbose', ['[DB Cleanup] Shrink Database'])
self.sql.execute ("VACUUM;")
self.commitDB()
#-------------------------------------------------------------------------------
def upgradeDB(self):

3
pialert/initialise.py
View File

@@ -175,8 +175,7 @@ def importConfigs (db):
# set these times to the past to force the first run
conf.last_internet_IP_scan = now_minus_24h
conf.last_scan_run = now_minus_24h
conf.last_cleanup = now_minus_24h
conf.last_scan_run = now_minus_24h
conf.last_update_vendors = conf.time_started - datetime.timedelta(days = 6) # update vendors 24h after first run and then once a week
conf.last_version_check = now_minus_24h