db_cleanup plugin

2025-12-06 17:15:38 -08:00 · 2023-09-17 21:46:05 +10:00
parent 80de181827
commit a5b952f18c
13 changed files with 370 additions and 119 deletions
--- a/front/plugins/db_cleanup/script.py
+++ b/front/plugins/db_cleanup/script.py
@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+# test script by running:
+# /home/pi/pialert/front/plugins/db_cleanup/script.py pluginskeephistory=250 hourstokeepnewdevice=48 daystokeepevents=90
+
+import os
+import pathlib
+import argparse
+import sys
+import hashlib
+import csv
+import sqlite3
+from io import StringIO
+from datetime import datetime
+
+sys.path.append("/home/pi/pialert/front/plugins")
+sys.path.append('/home/pi/pialert/pialert') 
+
+from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
+from logger import mylog, append_line_to_file
+from helper import timeNowTZ
+from const import logPath, pialertPath
+
+
+CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
+LOG_FILE = os.path.join(CUR_PATH, 'script.log')
+RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
+
+def main():
+    
+    parser = argparse.ArgumentParser(description='DB cleanup tasks')
+    parser.add_argument('pluginskeephistory', action="store", help="TBC")
+    parser.add_argument('hourstokeepnewdevice', action="store", help="TBC")
+    parser.add_argument('daystokeepevents', action="store", help="TBC")
+    parser.add_argument('pholuskeepdays', action="store", help="TBC")
+    
+    values = parser.parse_args()
+
+    PLUGINS_KEEP_HIST     = values.pluginskeephistory.split('=')[1]
+    HRS_TO_KEEP_NEWDEV    = values.hourstokeepnewdevice.split('=')[1]
+    DAYS_TO_KEEP_EVENTS   = values.daystokeepevents.split('=')[1]
+    PHOLUS_DAYS_DATA      = values.pholuskeepdays.split('=')[1]
+
+    mylog('verbose', ['[DBCLNP] In script'])     
+
+
+    # Execute cleanup/upkeep    
+    cleanup_database('/home/pi/pialert/db/pialert.db', DAYS_TO_KEEP_EVENTS, PHOLUS_DAYS_DATA, HRS_TO_KEEP_NEWDEV, PLUGINS_KEEP_HIST)
+    
+    mylog('verbose', ['[DBCLNP] Cleanup complete file '])   
+    
+    return 0
+
+#===============================================================================
+# Cleanup / upkeep database
+#===============================================================================
+def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, PHOLUS_DAYS_DATA, HRS_TO_KEEP_NEWDEV, PLUGINS_KEEP_HIST):
+    """
+    Cleaning out old records from the tables that don't need to keep all data.
+    """
+
+    mylog('verbose', ['[DBCLNP] Upkeep Database:' ])
+
+    # Connect to the PiAlert SQLite database
+    conn    = sqlite3.connect(dbPath)
+    cursor  = conn.cursor()
+
+    # Cleanup Online History
+    mylog('verbose', ['[DBCLNP] Online_History: Delete all but keep latest 150 entries'])
+    cursor.execute ("""DELETE from Online_History where "Index" not in (
+                            SELECT "Index" from Online_History 
+                            order by Scan_Date desc limit 150)""")
+    mylog('verbose', ['[DBCLNP] Optimize Database'])
+    # Cleanup Events
+    mylog('verbose', [f'[DBCLNP] Events: Delete all older than {str(DAYS_TO_KEEP_EVENTS)} days (DAYS_TO_KEEP_EVENTS setting)'])
+    cursor.execute (f"""DELETE FROM Events 
+                            WHERE eve_DateTime <= date('now', '-{str(DAYS_TO_KEEP_EVENTS)} day')""")
+                            
+    # Trim Plugins_History entries to less than PLUGINS_KEEP_HIST setting per unique "Plugin" column entry
+    mylog('verbose', [f'[DBCLNP] Plugins_History: Trim Plugins_History entries to less than {str(PLUGINS_KEEP_HIST)} per Plugin (PLUGINS_KEEP_HIST setting)'])
+
+    # Build the SQL query to delete entries that exceed the limit per unique "Plugin" column entry
+    delete_query = f"""DELETE FROM Plugins_History 
+                            WHERE "Index" NOT IN (
+                                SELECT "Index"
+                                FROM (
+                                    SELECT "Index", 
+                                        ROW_NUMBER() OVER(PARTITION BY "Plugin" ORDER BY DateTimeChanged DESC) AS row_num
+                                    FROM Plugins_History
+                                ) AS ranked_objects
+                                WHERE row_num <= {str(PLUGINS_KEEP_HIST)}
+                            );"""
+
+    cursor.execute(delete_query)
+
+    # Cleanup Pholus_Scan
+    if PHOLUS_DAYS_DATA != 0:
+        mylog('verbose', ['[DBCLNP] Pholus_Scan: Delete all older than ' + str(PHOLUS_DAYS_DATA) + ' days (PHOLUS_DAYS_DATA setting)'])
+        # todo: improvement possibility: keep at least N per mac
+        cursor.execute (f"""DELETE FROM Pholus_Scan 
+                                WHERE Time <= date('now', '-{str(PHOLUS_DAYS_DATA)} day')""") 
+    # Cleanup New Devices
+    if HRS_TO_KEEP_NEWDEV != 0:
+        mylog('verbose', [f'[DBCLNP] Devices: Delete all New Devices older than {str(HRS_TO_KEEP_NEWDEV)} hours (HRS_TO_KEEP_NEWDEV setting)'])            
+        cursor.execute (f"""DELETE FROM Devices 
+                                WHERE dev_NewDevice = 1 AND dev_FirstConnection < date('now', '+{str(HRS_TO_KEEP_NEWDEV)} hour')""") 
+
+
+    # De-dupe (de-duplicate) from the Plugins_Objects table 
+    # TODO This shouldn't be necessary - probably a concurrency bug somewhere in the code :(        
+    mylog('verbose', ['[DBCLNP] Plugins_Objects: Delete all duplicates'])
+    cursor.execute("""
+        DELETE FROM Plugins_Objects
+        WHERE rowid > (
+            SELECT MIN(rowid) FROM Plugins_Objects p2
+            WHERE Plugins_Objects.Plugin = p2.Plugin
+            AND Plugins_Objects.Object_PrimaryID = p2.Object_PrimaryID
+            AND Plugins_Objects.Object_SecondaryID = p2.Object_SecondaryID
+            AND Plugins_Objects.UserData = p2.UserData
+        )
+    """)
+
+    # De-Dupe (de-duplicate - remove duplicate entries) from the Pholus_Scan table
+    mylog('verbose', ['[DBCLNP] Pholus_Scan: Delete all duplicates'])
+    cursor.execute ("""DELETE  FROM Pholus_Scan
+                    WHERE rowid > (
+                    SELECT MIN(rowid) FROM Pholus_Scan p2
+                    WHERE Pholus_Scan.MAC = p2.MAC
+                    AND Pholus_Scan.Value = p2.Value
+                    AND Pholus_Scan.Record_Type = p2.Record_Type
+                    );""")
+
+    conn.commit()
+
+    # Shrink DB
+    mylog('verbose', ['[DBCLNP] Shrink Database'])
+    cursor.execute ("VACUUM;")
+
+    # Close the database connection
+    conn.close()
+    
+    
+
+#===============================================================================
+# BEGIN
+#===============================================================================
+if __name__ == '__main__':
+    main()