diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index d7f4f0a7..194c9c6e 100755 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -48,6 +48,36 @@ Two plugins help maintain the system’s performance: --- +## Database Performance Tuning + +The application automatically maintains database performance as data accumulates. However, you can adjust settings to balance CPU usage, disk usage, and responsiveness. + +### **WAL Size Tuning (Storage vs. CPU Tradeoff)** + +The SQLite Write-Ahead Log (WAL) is a temporary file that grows during normal operation. On systems with constrained resources (NAS, Raspberry Pi), controlling WAL size is important. + +**Setting:** **Settings → General → "WAL size limit (MB)"** (default: **50 MB**) + +| Setting | Effect | Use Case | +|---------|--------|----------| +| **10–20 MB** | Smaller storage footprint; more frequent disk operations | NAS with SD card (storage priority) | +| **50 MB** (default) | Balanced; recommended for most setups | General use | +| **75–100 MB** | Smoother performance; larger WAL on disk | High-speed NAS or servers | + +**Recommendation:** For NAS devices with SD cards, leave at default (50 MB) or increase slightly (75 MB). Avoid very low values (< 10 MB) as they cause frequent disk thrashing and CPU spikes. + +### **Automatic Cleanup** + +The DB cleanup plugin (`DBCLNP`) automatically optimizes query performance and trims old data: + +- **Deletes old events** – Controlled by `DAYS_TO_KEEP_EVENTS` (default: 90 days) +- **Trims plugin history** – Keeps recent entries only (controlled by `PLUGINS_KEEP_HIST`) +- **Optimizes queries** – Updates database statistics so queries remain fast + +**If cleanup fails**, performance degrades quickly. Check **Maintenance → Logs** for errors. If you see frequent failures, increase the timeout (`DBCLNP_RUN_TIMEOUT`). + +--- + ## Scan Frequency and Coverage Frequent scans increase resource usage, network traffic, and database read/write cycles. diff --git a/front/php/templates/language/en_us.json b/front/php/templates/language/en_us.json index 9f2d6af1..3de49137 100755 --- a/front/php/templates/language/en_us.json +++ b/front/php/templates/language/en_us.json @@ -63,6 +63,8 @@ "BackDevices_darkmode_enabled": "Darkmode Enabled", "CLEAR_NEW_FLAG_description": "If enabled (0 is disabled), devices flagged as New Device will be unflagged if the time limit (specified in hours) exceeds their First Session time.", "CLEAR_NEW_FLAG_name": "Clear new flag", + "PRAGMA_JOURNAL_SIZE_LIMIT_description": "SQLite WAL (Write-Ahead Log) maximum size in MB before triggering automatic checkpoints. Lower values (10-20 MB) reduce disk/storage usage but increase CPU usage during scans. Higher values (50-100 MB) reduce CPU spikes during operations but may use more RAM and disk space. Default 50 MB balances both. Useful for resource-constrained systems like NAS devices with SD cards.", + "PRAGMA_JOURNAL_SIZE_LIMIT_name": "WAL size limit (MB)", "CustProps_cant_remove": "Can't remove, at least one property is needed.", "DAYS_TO_KEEP_EVENTS_description": "This is a maintenance setting. This specifies the number of days worth of event entries that will be kept. All older events will be deleted periodically. Also applies on Plugin Events History.", "DAYS_TO_KEEP_EVENTS_name": "Delete events older than", diff --git a/server/database.py b/server/database.py index 477cfc2b..405c47a8 100755 --- a/server/database.py +++ b/server/database.py @@ -75,10 +75,17 @@ class DB: # When temp_store is MEMORY (2) temporary tables and indices # are kept as if they were in pure in-memory databases. self.sql_connection.execute("PRAGMA temp_store=MEMORY;") - # WAL size limit: cap at 10 MB. When approached, SQLite auto-checkpoints + # WAL size limit: auto-checkpoint when WAL approaches this size, # even if other connections are active. Prevents unbounded WAL growth # on systems with multiple long-lived processes (backend, nginx, PHP-FPM). - self.sql_connection.execute("PRAGMA journal_size_limit=10000000;") + # User-configurable via PRAGMA_JOURNAL_SIZE_LIMIT setting (default 50 MB). + try: + from helper import get_setting_value + wal_limit_mb = int(get_setting_value("PRAGMA_JOURNAL_SIZE_LIMIT", "50")) + wal_limit_bytes = wal_limit_mb * 1000000 + except Exception: + wal_limit_bytes = 50000000 # 50 MB fallback + self.sql_connection.execute(f"PRAGMA journal_size_limit={wal_limit_bytes};") self.sql_connection.text_factory = str self.sql_connection.row_factory = sqlite3.Row @@ -334,6 +341,13 @@ def get_temp_db_connection(): conn = sqlite3.connect(fullDbPath, timeout=5, isolation_level=None) conn.execute("PRAGMA journal_mode=WAL;") conn.execute("PRAGMA busy_timeout=5000;") # 5s wait before giving up - conn.execute("PRAGMA journal_size_limit=10000000;") # 10 MB WAL cap with auto-checkpoint + # Apply user-configured WAL size limit (default 50 MB in initialise.py) + try: + from helper import get_setting_value + wal_limit_mb = int(get_setting_value("PRAGMA_JOURNAL_SIZE_LIMIT", "50")) + wal_limit_bytes = wal_limit_mb * 1000000 + except Exception: + wal_limit_bytes = 50000000 # 50 MB fallback + conn.execute(f"PRAGMA journal_size_limit={wal_limit_bytes};") conn.row_factory = sqlite3.Row return conn diff --git a/server/initialise.py b/server/initialise.py index b11eb40a..afed6dd8 100755 --- a/server/initialise.py +++ b/server/initialise.py @@ -341,6 +341,15 @@ def importConfigs(pm, db, all_plugins): "[]", "General", ) + conf.PRAGMA_JOURNAL_SIZE_LIMIT = ccd( + "PRAGMA_JOURNAL_SIZE_LIMIT", + 50, + c_d, + "WAL size limit (MB)", + '{"dataType":"integer", "elements": [{"elementType" : "input", "elementOptions" : [{"type": "number"}] ,"transformers": []}]}', + "[]", + "General", + ) conf.REFRESH_FQDN = ccd( "REFRESH_FQDN", False,