vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2025-12-06 17:15:38 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

/data and /tmp standarization

Browse Source
This commit is contained in:
Adam Outler
2025-11-04 22:26:35 +00:00
parent 90a07c61eb
commit 5b871865db
250 changed files with 7462 additions and 4940 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
front/plugins/db_cleanup/script.py
View File

@@ -1,88 +1,110 @@
#!/usr/bin/env python
import os
import pathlib
import argparse
import sys
import hashlib
import csv
import sqlite3
from io import StringIO
from datetime import datetime
# Register NetAlertX directories
INSTALL_PATH="/app"
INSTALL_PATH = os.getenv("NETALERTX_APP", "/app")
sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"])
from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
from logger import mylog, Logger, append_line_to_file
from helper import timeNowTZ, get_setting_value
from const import logPath, applicationPath, fullDbPath
from logger import mylog, Logger
from helper import get_setting_value
from const import logPath, fullDbPath
import conf
from pytz import timezone
# Make sure the TIMEZONE for logging is correct
conf.tz = timezone(get_setting_value('TIMEZONE'))
conf.tz = timezone(get_setting_value("TIMEZONE"))
# Make sure log level is initialized correctly
Logger(get_setting_value('LOG_LEVEL'))
Logger(get_setting_value("LOG_LEVEL"))
pluginName = 'DBCLNP'
LOG_PATH = logPath + '/plugins'
LOG_FILE = os.path.join(LOG_PATH, f'script.{pluginName}.log')
RESULT_FILE = os.path.join(LOG_PATH, f'last_result.{pluginName}.log')
pluginName = "DBCLNP"
LOG_PATH = logPath + "/plugins"
LOG_FILE = os.path.join(LOG_PATH, f"script.{pluginName}.log")
RESULT_FILE = os.path.join(LOG_PATH, f"last_result.{pluginName}.log")
def main():
PLUGINS_KEEP_HIST = int(get_setting_value("PLUGINS_KEEP_HIST"))
HRS_TO_KEEP_NEWDEV = int(get_setting_value("HRS_TO_KEEP_NEWDEV"))
HRS_TO_KEEP_OFFDEV = int(get_setting_value("HRS_TO_KEEP_OFFDEV"))
DAYS_TO_KEEP_EVENTS = int(get_setting_value("DAYS_TO_KEEP_EVENTS"))
CLEAR_NEW_FLAG = get_setting_value("CLEAR_NEW_FLAG")
mylog('verbose', [f'[{pluginName}] In script'])
PLUGINS_KEEP_HIST = int(get_setting_value("PLUGINS_KEEP_HIST"))
HRS_TO_KEEP_NEWDEV = int(get_setting_value("HRS_TO_KEEP_NEWDEV"))
HRS_TO_KEEP_OFFDEV = int(get_setting_value("HRS_TO_KEEP_OFFDEV"))
DAYS_TO_KEEP_EVENTS = int(get_setting_value("DAYS_TO_KEEP_EVENTS"))
CLEAR_NEW_FLAG = get_setting_value("CLEAR_NEW_FLAG")
mylog("verbose", [f"[{pluginName}] In script"])
# Execute cleanup/upkeep
cleanup_database(
fullDbPath,
DAYS_TO_KEEP_EVENTS,
HRS_TO_KEEP_NEWDEV,
HRS_TO_KEEP_OFFDEV,
PLUGINS_KEEP_HIST,
CLEAR_NEW_FLAG,
)
mylog("verbose", [f"[{pluginName}] Cleanup complete"])
# Execute cleanup/upkeep
cleanup_database(fullDbPath, DAYS_TO_KEEP_EVENTS, HRS_TO_KEEP_NEWDEV, HRS_TO_KEEP_OFFDEV, PLUGINS_KEEP_HIST, CLEAR_NEW_FLAG)
mylog('verbose', [f'[{pluginName}] Cleanup complete'])
return 0
#===============================================================================
# ===============================================================================
# Cleanup / upkeep database
#===============================================================================
def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, HRS_TO_KEEP_NEWDEV, HRS_TO_KEEP_OFFDEV, PLUGINS_KEEP_HIST, CLEAR_NEW_FLAG):
# ===============================================================================
def cleanup_database(
dbPath,
DAYS_TO_KEEP_EVENTS,
HRS_TO_KEEP_NEWDEV,
HRS_TO_KEEP_OFFDEV,
PLUGINS_KEEP_HIST,
CLEAR_NEW_FLAG,
):
"""
Cleaning out old records from the tables that don't need to keep all data.
"""
mylog('verbose', [f'[{pluginName}] Upkeep Database:' ])
mylog("verbose", [f"[{pluginName}] Upkeep Database:"])
# Connect to the App database
conn = sqlite3.connect(dbPath, timeout=30)
cursor = conn.cursor()
conn = sqlite3.connect(dbPath, timeout=30)
cursor = conn.cursor()
# -----------------------------------------------------
# Cleanup Online History
mylog('verbose', [f'[{pluginName}] Online_History: Delete all but keep latest 150 entries'])
cursor.execute ("""DELETE from Online_History where "Index" not in (
mylog(
"verbose",
[f"[{pluginName}] Online_History: Delete all but keep latest 150 entries"],
)
cursor.execute(
"""DELETE from Online_History where "Index" not in (
SELECT "Index" from Online_History
order by Scan_Date desc limit 150)""")
order by Scan_Date desc limit 150)"""
)
# -----------------------------------------------------
# Cleanup Events
mylog('verbose', [f'[{pluginName}] Events: Delete all older than {str(DAYS_TO_KEEP_EVENTS)} days (DAYS_TO_KEEP_EVENTS setting)'])
cursor.execute (f"""DELETE FROM Events
WHERE eve_DateTime <= date('now', '-{str(DAYS_TO_KEEP_EVENTS)} day')""")
# -----------------------------------------------------
mylog(
"verbose",
[
f"[{pluginName}] Events: Delete all older than {str(DAYS_TO_KEEP_EVENTS)} days (DAYS_TO_KEEP_EVENTS setting)"
],
)
cursor.execute(
f"""DELETE FROM Events
WHERE eve_DateTime <= date('now', '-{str(DAYS_TO_KEEP_EVENTS)} day')"""
)
# -----------------------------------------------------
# Trim Plugins_History entries to less than PLUGINS_KEEP_HIST setting per unique "Plugin" column entry
mylog('verbose', [f'[{pluginName}] Plugins_History: Trim Plugins_History entries to less than {str(PLUGINS_KEEP_HIST)} per Plugin (PLUGINS_KEEP_HIST setting)'])
mylog(
"verbose",
[
f"[{pluginName}] Plugins_History: Trim Plugins_History entries to less than {str(PLUGINS_KEEP_HIST)} per Plugin (PLUGINS_KEEP_HIST setting)"
],
)
# Build the SQL query to delete entries that exceed the limit per unique "Plugin" column entry
delete_query = f"""DELETE FROM Plugins_History
@@ -101,11 +123,16 @@ def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, HRS_TO_KEEP_NEWDEV, HRS_TO_KE
# -----------------------------------------------------
# Trim Notifications entries to less than DBCLNP_NOTIFI_HIST setting
histCount = get_setting_value('DBCLNP_NOTIFI_HIST')
histCount = get_setting_value("DBCLNP_NOTIFI_HIST")
mylog('verbose', [f'[{pluginName}] Plugins_History: Trim Notifications entries to less than {histCount}'])
mylog(
"verbose",
[
f"[{pluginName}] Plugins_History: Trim Notifications entries to less than {histCount}"
],
)
# Build the SQL query to delete entries
# Build the SQL query to delete entries
delete_query = f"""DELETE FROM Notifications
WHERE "Index" NOT IN (
SELECT "Index"
@@ -119,14 +146,13 @@ def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, HRS_TO_KEEP_NEWDEV, HRS_TO_KE
cursor.execute(delete_query)
# -----------------------------------------------------
# Trim Workflow entries to less than WORKFLOWS_AppEvents_hist setting
histCount = get_setting_value('WORKFLOWS_AppEvents_hist')
histCount = get_setting_value("WORKFLOWS_AppEvents_hist")
mylog('verbose', [f'[{pluginName}] Trim AppEvents to less than {histCount}'])
mylog("verbose", [f"[{pluginName}] Trim AppEvents to less than {histCount}"])
# Build the SQL query to delete entries
# Build the SQL query to delete entries
delete_query = f"""DELETE FROM AppEvents
WHERE "Index" NOT IN (
SELECT "Index"
@@ -141,38 +167,52 @@ def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, HRS_TO_KEEP_NEWDEV, HRS_TO_KE
cursor.execute(delete_query)
conn.commit()
# -----------------------------------------------------
# Cleanup New Devices
if HRS_TO_KEEP_NEWDEV != 0:
mylog('verbose', [f'[{pluginName}] Devices: Delete all New Devices older than {str(HRS_TO_KEEP_NEWDEV)} hours (HRS_TO_KEEP_NEWDEV setting)'])
mylog(
"verbose",
[
f"[{pluginName}] Devices: Delete all New Devices older than {str(HRS_TO_KEEP_NEWDEV)} hours (HRS_TO_KEEP_NEWDEV setting)"
],
)
query = f"""DELETE FROM Devices WHERE devIsNew = 1 AND devFirstConnection < date('now', '-{str(HRS_TO_KEEP_NEWDEV)} hour')"""
mylog('verbose', [f'[{pluginName}] Query: {query} '])
cursor.execute (query)
mylog("verbose", [f"[{pluginName}] Query: {query} "])
cursor.execute(query)
# -----------------------------------------------------
# Cleanup Offline Devices
if HRS_TO_KEEP_OFFDEV != 0:
mylog('verbose', [f'[{pluginName}] Devices: Delete all New Devices older than {str(HRS_TO_KEEP_OFFDEV)} hours (HRS_TO_KEEP_OFFDEV setting)'])
mylog(
"verbose",
[
f"[{pluginName}] Devices: Delete all New Devices older than {str(HRS_TO_KEEP_OFFDEV)} hours (HRS_TO_KEEP_OFFDEV setting)"
],
)
query = f"""DELETE FROM Devices WHERE devPresentLastScan = 0 AND devLastConnection < date('now', '-{str(HRS_TO_KEEP_OFFDEV)} hour')"""
mylog('verbose', [f'[{pluginName}] Query: {query} '])
cursor.execute (query)
mylog("verbose", [f"[{pluginName}] Query: {query} "])
cursor.execute(query)
# -----------------------------------------------------
# Clear New Flag
if CLEAR_NEW_FLAG != 0:
mylog('verbose', [f'[{pluginName}] Devices: Clear "New Device" flag for all devices older than {str(CLEAR_NEW_FLAG)} hours (CLEAR_NEW_FLAG setting)'])
mylog(
"verbose",
[
f'[{pluginName}] Devices: Clear "New Device" flag for all devices older than {str(CLEAR_NEW_FLAG)} hours (CLEAR_NEW_FLAG setting)'
],
)
query = f"""UPDATE Devices SET devIsNew = 0 WHERE devIsNew = 1 AND date(devFirstConnection, '+{str(CLEAR_NEW_FLAG)} hour') < date('now')"""
# select * from Devices where devIsNew = 1 AND date(devFirstConnection, '+3 hour' ) < date('now')
mylog('verbose', [f'[{pluginName}] Query: {query} '])
# select * from Devices where devIsNew = 1 AND date(devFirstConnection, '+3 hour' ) < date('now')
mylog("verbose", [f"[{pluginName}] Query: {query} "])
cursor.execute(query)
# -----------------------------------------------------
# De-dupe (de-duplicate) from the Plugins_Objects table
# TODO This shouldn't be necessary - probably a concurrency bug somewhere in the code :(
mylog('verbose', [f'[{pluginName}] Plugins_Objects: Delete all duplicates'])
cursor.execute("""
# De-dupe (de-duplicate) from the Plugins_Objects table
# TODO This shouldn't be necessary - probably a concurrency bug somewhere in the code :(
mylog("verbose", [f"[{pluginName}] Plugins_Objects: Delete all duplicates"])
cursor.execute(
"""
DELETE FROM Plugins_Objects
WHERE rowid > (
SELECT MIN(rowid) FROM Plugins_Objects p2
@@ -181,8 +221,8 @@ def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, HRS_TO_KEEP_NEWDEV, HRS_TO_KE
AND Plugins_Objects.Object_SecondaryID = p2.Object_SecondaryID
AND Plugins_Objects.UserData = p2.UserData
)
""")
"""
)
conn.commit()
@@ -190,18 +230,18 @@ def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, HRS_TO_KEEP_NEWDEV, HRS_TO_KE
cursor.execute("PRAGMA wal_checkpoint(TRUNCATE);")
cursor.execute("PRAGMA wal_checkpoint(FULL);")
mylog('verbose', [f'[{pluginName}] WAL checkpoint executed to truncate file.'])
mylog("verbose", [f"[{pluginName}] WAL checkpoint executed to truncate file."])
# Shrink DB
mylog('verbose', [f'[{pluginName}] Shrink Database'])
cursor.execute ("VACUUM;")
mylog("verbose", [f"[{pluginName}] Shrink Database"])
cursor.execute("VACUUM;")
# Close the database connection
conn.close()
conn.close()
#===============================================================================
# ===============================================================================
# BEGIN
#===============================================================================
if __name__ == '__main__':
main()
# ===============================================================================
if __name__ == "__main__":
main()