mirror of
https://github.com/jokob-sk/NetAlertX.git
synced 2025-12-07 01:26:11 -08:00
db_cleanup plugin
This commit is contained in:
Jokob-sk
7
front/plugins/db_cleanup/README.md
Executable file
7
front/plugins/db_cleanup/README.md
Executable file
@@ -0,0 +1,7 @@
|
||||
## Overview
|
||||
|
||||
Plugin to run regular database cleanup tasks. It is strongly recommended to have an hourly or at least daily schedule running.
|
||||
|
||||
### Usage
|
||||
|
||||
- Check the Settings page for details.
|
||||
180
front/plugins/db_cleanup/config.json
Executable file
180
front/plugins/db_cleanup/config.json
Executable file
@@ -0,0 +1,180 @@
|
||||
{
|
||||
"code_name": "db_cleanup",
|
||||
"unique_prefix": "DBCLNP",
|
||||
"enabled": true,
|
||||
"data_source": "script",
|
||||
"show_ui": false,
|
||||
"localized": ["display_name", "description", "icon"],
|
||||
|
||||
"display_name": [
|
||||
{
|
||||
"language_code": "en_us",
|
||||
"string": "DB cleanup"
|
||||
}
|
||||
],
|
||||
"icon": [
|
||||
{
|
||||
"language_code": "en_us",
|
||||
"string": "<i class=\"fa-solid fa-broom\"></i>"
|
||||
}
|
||||
],
|
||||
"description": [
|
||||
{
|
||||
"language_code": "en_us",
|
||||
"string": "A plugin to schedule database cleanup & upkeep tasks."
|
||||
}
|
||||
],
|
||||
"params" : [{
|
||||
"name" : "pluginskeephistory",
|
||||
"type" : "setting",
|
||||
"value" : "PLUGINS_KEEP_HIST"
|
||||
},
|
||||
{
|
||||
"name" : "daystokeepevents",
|
||||
"type" : "setting",
|
||||
"value" : "DAYS_TO_KEEP_EVENTS"
|
||||
},
|
||||
{
|
||||
"name" : "hourstokeepnewdevice",
|
||||
"type" : "setting",
|
||||
"value" : "HRS_TO_KEEP_NEWDEV"
|
||||
},
|
||||
{
|
||||
"name" : "pholuskeepdays",
|
||||
"type" : "setting",
|
||||
"value" : "PHOLUS_DAYS_DATA"
|
||||
}
|
||||
],
|
||||
|
||||
"settings": [
|
||||
{
|
||||
"function": "RUN",
|
||||
"type": "text.select",
|
||||
"default_value":"schedule",
|
||||
"options": ["disabled", "once", "schedule", "always_after_scan"],
|
||||
"localized": ["name", "description"],
|
||||
"name" :[{
|
||||
"language_code":"en_us",
|
||||
"string" : "When to run"
|
||||
},
|
||||
{
|
||||
"language_code":"es_es",
|
||||
"string" : "Cuándo ejecutar"
|
||||
},
|
||||
{
|
||||
"language_code":"de_de",
|
||||
"string" : "Wann laufen"
|
||||
}],
|
||||
"description": [{
|
||||
"language_code":"en_us",
|
||||
"string" : "When the cleanup should be performed. An hourly or daily <code>SCHEDULE</code> is a good option."
|
||||
}]
|
||||
},
|
||||
{
|
||||
"function": "CMD",
|
||||
"type": "readonly",
|
||||
"default_value": "python3 /home/pi/pialert/front/plugins/db_cleanup/script.py pluginskeephistory={pluginskeephistory} hourstokeepnewdevice={hourstokeepnewdevice} daystokeepevents={daystokeepevents} pholuskeepdays={pholuskeepdays}",
|
||||
"options": [],
|
||||
"localized": ["name", "description"],
|
||||
"name": [
|
||||
{
|
||||
"language_code": "en_us",
|
||||
"string": "Command"
|
||||
},
|
||||
{
|
||||
"language_code": "es_es",
|
||||
"string": "Comando"
|
||||
},
|
||||
{
|
||||
"language_code": "de_de",
|
||||
"string": "Befehl"
|
||||
}
|
||||
],
|
||||
"description": [
|
||||
{
|
||||
"language_code": "en_us",
|
||||
"string": "Command to run. This can not be changed"
|
||||
},
|
||||
{
|
||||
"language_code": "es_es",
|
||||
"string": "Comando a ejecutar. Esto no se puede cambiar"
|
||||
},
|
||||
{
|
||||
"language_code": "de_de",
|
||||
"string": "Befehl zum Ausführen. Dies kann nicht geändert werden"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"function": "RUN_SCHD",
|
||||
"type": "text",
|
||||
"default_value":"*/30 * * * *",
|
||||
"options": [],
|
||||
"localized": ["name", "description"],
|
||||
"name" : [{
|
||||
"language_code":"en_us",
|
||||
"string" : "Schedule"
|
||||
},
|
||||
{
|
||||
"language_code":"es_es",
|
||||
"string" : "Schedule"
|
||||
},
|
||||
{
|
||||
"language_code":"de_de",
|
||||
"string" : "Schedule"
|
||||
}],
|
||||
"description": [{
|
||||
"language_code":"en_us",
|
||||
"string" : "Only enabled if you select <code>schedule</code> in the <a href=\"#CSVBCKP_RUN\"><code>CSVBCKP_RUN</code> setting</a>. Make sure you enter the schedule in the correct cron-like format (e.g. validate at <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). For example entering <code>0 4 * * *</code> will run the scan after 4 am in the <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</code> you set above</a>. Will be run NEXT time the time passes."
|
||||
},
|
||||
{
|
||||
"language_code":"es_es",
|
||||
"string" : "Solo está habilitado si selecciona <code>schedule</code> en la configuración <a href=\"#CSVBCKP_RUN\"><code>CSVBCKP_RUN</code></a>. Asegúrese de ingresar la programación en el formato similar a cron correcto (por ejemplo, valide en <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Por ejemplo, ingresar <code>0 4 * * *</code> ejecutará el escaneo después de las 4 a.m. en el <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ código> que configuró arriba</a>. Se ejecutará la PRÓXIMA vez que pase el tiempo."
|
||||
},
|
||||
{
|
||||
"language_code":"de_de",
|
||||
"string" : "Nur aktiviert, wenn Sie <code>schedule</code> in der <a href=\"#CSVBCKP_RUN\"><code>CSVBCKP_RUN</code>-Einstellung</a> auswählen. Stellen Sie sicher, dass Sie den Zeitplan im richtigen Cron-ähnlichen Format eingeben (z. B. validieren unter <a href=\"https://crontab.guru/\" target=\"_blank\">crontab.guru</a>). Wenn Sie beispielsweise <code>0 4 * * *</code> eingeben, wird der Scan nach 4 Uhr morgens in der <a onclick=\"toggleAllSettings()\" href=\"#TIMEZONE\"><code>TIMEZONE</ ausgeführt. Code> den Sie oben festgelegt haben</a>. Wird das NÄCHSTE Mal ausgeführt, wenn die Zeit vergeht."
|
||||
}]
|
||||
},
|
||||
{
|
||||
"function": "RUN_TIMEOUT",
|
||||
"type": "integer",
|
||||
"default_value": 30,
|
||||
"options": [],
|
||||
"localized": ["name", "description"],
|
||||
"name": [
|
||||
{
|
||||
"language_code": "en_us",
|
||||
"string": "Run timeout"
|
||||
},
|
||||
{
|
||||
"language_code": "es_es",
|
||||
"string": "Tiempo límite de ejecución"
|
||||
},
|
||||
{
|
||||
"language_code": "de_de",
|
||||
"string": "Zeitüberschreitung"
|
||||
}
|
||||
],
|
||||
"description": [
|
||||
{
|
||||
"language_code": "en_us",
|
||||
"string": "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted."
|
||||
},
|
||||
{
|
||||
"language_code": "es_es",
|
||||
"string": "Tiempo máximo en segundos para esperar a que finalice el script. Si se supera este tiempo, el script se cancela."
|
||||
},
|
||||
{
|
||||
"language_code": "de_de",
|
||||
"string": "Maximale Zeit in Sekunden, die auf den Abschluss des Skripts gewartet werden soll. Bei Überschreitung dieser Zeit wird das Skript abgebrochen."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
"database_column_definitions":
|
||||
[
|
||||
|
||||
]
|
||||
}
|
||||
147
front/plugins/db_cleanup/script.py
Executable file
147
front/plugins/db_cleanup/script.py
Executable file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python
|
||||
# test script by running:
|
||||
# /home/pi/pialert/front/plugins/db_cleanup/script.py pluginskeephistory=250 hourstokeepnewdevice=48 daystokeepevents=90
|
||||
|
||||
import os
|
||||
import pathlib
|
||||
import argparse
|
||||
import sys
|
||||
import hashlib
|
||||
import csv
|
||||
import sqlite3
|
||||
from io import StringIO
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.append("/home/pi/pialert/front/plugins")
|
||||
sys.path.append('/home/pi/pialert/pialert')
|
||||
|
||||
from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
|
||||
from logger import mylog, append_line_to_file
|
||||
from helper import timeNowTZ
|
||||
from const import logPath, pialertPath
|
||||
|
||||
|
||||
CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
|
||||
LOG_FILE = os.path.join(CUR_PATH, 'script.log')
|
||||
RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser(description='DB cleanup tasks')
|
||||
parser.add_argument('pluginskeephistory', action="store", help="TBC")
|
||||
parser.add_argument('hourstokeepnewdevice', action="store", help="TBC")
|
||||
parser.add_argument('daystokeepevents', action="store", help="TBC")
|
||||
parser.add_argument('pholuskeepdays', action="store", help="TBC")
|
||||
|
||||
values = parser.parse_args()
|
||||
|
||||
PLUGINS_KEEP_HIST = values.pluginskeephistory.split('=')[1]
|
||||
HRS_TO_KEEP_NEWDEV = values.hourstokeepnewdevice.split('=')[1]
|
||||
DAYS_TO_KEEP_EVENTS = values.daystokeepevents.split('=')[1]
|
||||
PHOLUS_DAYS_DATA = values.pholuskeepdays.split('=')[1]
|
||||
|
||||
mylog('verbose', ['[DBCLNP] In script'])
|
||||
|
||||
|
||||
# Execute cleanup/upkeep
|
||||
cleanup_database('/home/pi/pialert/db/pialert.db', DAYS_TO_KEEP_EVENTS, PHOLUS_DAYS_DATA, HRS_TO_KEEP_NEWDEV, PLUGINS_KEEP_HIST)
|
||||
|
||||
mylog('verbose', ['[DBCLNP] Cleanup complete file '])
|
||||
|
||||
return 0
|
||||
|
||||
#===============================================================================
|
||||
# Cleanup / upkeep database
|
||||
#===============================================================================
|
||||
def cleanup_database (dbPath, DAYS_TO_KEEP_EVENTS, PHOLUS_DAYS_DATA, HRS_TO_KEEP_NEWDEV, PLUGINS_KEEP_HIST):
|
||||
"""
|
||||
Cleaning out old records from the tables that don't need to keep all data.
|
||||
"""
|
||||
|
||||
mylog('verbose', ['[DBCLNP] Upkeep Database:' ])
|
||||
|
||||
# Connect to the PiAlert SQLite database
|
||||
conn = sqlite3.connect(dbPath)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Cleanup Online History
|
||||
mylog('verbose', ['[DBCLNP] Online_History: Delete all but keep latest 150 entries'])
|
||||
cursor.execute ("""DELETE from Online_History where "Index" not in (
|
||||
SELECT "Index" from Online_History
|
||||
order by Scan_Date desc limit 150)""")
|
||||
mylog('verbose', ['[DBCLNP] Optimize Database'])
|
||||
# Cleanup Events
|
||||
mylog('verbose', [f'[DBCLNP] Events: Delete all older than {str(DAYS_TO_KEEP_EVENTS)} days (DAYS_TO_KEEP_EVENTS setting)'])
|
||||
cursor.execute (f"""DELETE FROM Events
|
||||
WHERE eve_DateTime <= date('now', '-{str(DAYS_TO_KEEP_EVENTS)} day')""")
|
||||
|
||||
# Trim Plugins_History entries to less than PLUGINS_KEEP_HIST setting per unique "Plugin" column entry
|
||||
mylog('verbose', [f'[DBCLNP] Plugins_History: Trim Plugins_History entries to less than {str(PLUGINS_KEEP_HIST)} per Plugin (PLUGINS_KEEP_HIST setting)'])
|
||||
|
||||
# Build the SQL query to delete entries that exceed the limit per unique "Plugin" column entry
|
||||
delete_query = f"""DELETE FROM Plugins_History
|
||||
WHERE "Index" NOT IN (
|
||||
SELECT "Index"
|
||||
FROM (
|
||||
SELECT "Index",
|
||||
ROW_NUMBER() OVER(PARTITION BY "Plugin" ORDER BY DateTimeChanged DESC) AS row_num
|
||||
FROM Plugins_History
|
||||
) AS ranked_objects
|
||||
WHERE row_num <= {str(PLUGINS_KEEP_HIST)}
|
||||
);"""
|
||||
|
||||
cursor.execute(delete_query)
|
||||
|
||||
# Cleanup Pholus_Scan
|
||||
if PHOLUS_DAYS_DATA != 0:
|
||||
mylog('verbose', ['[DBCLNP] Pholus_Scan: Delete all older than ' + str(PHOLUS_DAYS_DATA) + ' days (PHOLUS_DAYS_DATA setting)'])
|
||||
# todo: improvement possibility: keep at least N per mac
|
||||
cursor.execute (f"""DELETE FROM Pholus_Scan
|
||||
WHERE Time <= date('now', '-{str(PHOLUS_DAYS_DATA)} day')""")
|
||||
# Cleanup New Devices
|
||||
if HRS_TO_KEEP_NEWDEV != 0:
|
||||
mylog('verbose', [f'[DBCLNP] Devices: Delete all New Devices older than {str(HRS_TO_KEEP_NEWDEV)} hours (HRS_TO_KEEP_NEWDEV setting)'])
|
||||
cursor.execute (f"""DELETE FROM Devices
|
||||
WHERE dev_NewDevice = 1 AND dev_FirstConnection < date('now', '+{str(HRS_TO_KEEP_NEWDEV)} hour')""")
|
||||
|
||||
|
||||
# De-dupe (de-duplicate) from the Plugins_Objects table
|
||||
# TODO This shouldn't be necessary - probably a concurrency bug somewhere in the code :(
|
||||
mylog('verbose', ['[DBCLNP] Plugins_Objects: Delete all duplicates'])
|
||||
cursor.execute("""
|
||||
DELETE FROM Plugins_Objects
|
||||
WHERE rowid > (
|
||||
SELECT MIN(rowid) FROM Plugins_Objects p2
|
||||
WHERE Plugins_Objects.Plugin = p2.Plugin
|
||||
AND Plugins_Objects.Object_PrimaryID = p2.Object_PrimaryID
|
||||
AND Plugins_Objects.Object_SecondaryID = p2.Object_SecondaryID
|
||||
AND Plugins_Objects.UserData = p2.UserData
|
||||
)
|
||||
""")
|
||||
|
||||
# De-Dupe (de-duplicate - remove duplicate entries) from the Pholus_Scan table
|
||||
mylog('verbose', ['[DBCLNP] Pholus_Scan: Delete all duplicates'])
|
||||
cursor.execute ("""DELETE FROM Pholus_Scan
|
||||
WHERE rowid > (
|
||||
SELECT MIN(rowid) FROM Pholus_Scan p2
|
||||
WHERE Pholus_Scan.MAC = p2.MAC
|
||||
AND Pholus_Scan.Value = p2.Value
|
||||
AND Pholus_Scan.Record_Type = p2.Record_Type
|
||||
);""")
|
||||
|
||||
conn.commit()
|
||||
|
||||
# Shrink DB
|
||||
mylog('verbose', ['[DBCLNP] Shrink Database'])
|
||||
cursor.execute ("VACUUM;")
|
||||
|
||||
# Close the database connection
|
||||
conn.close()
|
||||
|
||||
|
||||
|
||||
#===============================================================================
|
||||
# BEGIN
|
||||
#===============================================================================
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user