vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2026-03-31 07:12:23 -07:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

BE+FE: refactor timezone UTC #1506

Browse Source 
Signed-off-by: jokob-sk <jokob.sk@gmail.com>
This commit is contained in:
jokob-sk
2026-02-11 16:15:49 +11:00
parent 70c3530a5c
commit b57d36607a
4 changed files with 116 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
server/db/db_upgrade.py
View File

@@ -1,10 +1,6 @@
import sys
import os
# Register NetAlertX directories
INSTALL_PATH = os.getenv("NETALERTX_APP", "/app")
sys.path.extend([f"{INSTALL_PATH}/server"])
import conf
from zoneinfo import ZoneInfo
import datetime as dt
from logger import mylog # noqa: E402 [flake8 lint suppression]
from messaging.in_app import write_notification # noqa: E402 [flake8 lint suppression]
@@ -503,14 +499,14 @@ def ensure_plugins_tables(sql) -> bool:
def is_timestamps_in_utc(sql) -> bool:
"""
Check if existing timestamps in Devices table are already in UTC format.
Strategy:
1. Sample 10 non-NULL devFirstConnection timestamps from Devices
2. For each timestamp, assume it's UTC and calculate what it would be in local time
3. Check if timestamps have a consistent offset pattern (indicating local time storage)
4. If offset is consistently > 0, they're likely local timestamps (need migration)
5. If offset is ~0 or inconsistent, they're likely already UTC (skip migration)
Returns:
bool: True if timestamps appear to be in UTC already, False if they need migration
"""
@@ -519,10 +515,10 @@ def is_timestamps_in_utc(sql) -> bool:
import conf
from zoneinfo import ZoneInfo
import datetime as dt
now = dt.datetime.now(dt.UTC).replace(microsecond=0)
current_offset_seconds = 0
try:
if isinstance(conf.tz, dt.tzinfo):
tz = conf.tz
@@ -532,13 +528,13 @@ def is_timestamps_in_utc(sql) -> bool:
tz = None
except Exception:
tz = None
if tz:
local_now = dt.datetime.now(tz).replace(microsecond=0)
local_offset = local_now.utcoffset().total_seconds()
utc_offset = now.utcoffset().total_seconds() if now.utcoffset() else 0
current_offset_seconds = int(local_offset - utc_offset)
# Sample timestamps from Devices table
sql.execute("""
SELECT devFirstConnection, devLastConnection, devLastNotification
@@ -546,27 +542,27 @@ def is_timestamps_in_utc(sql) -> bool:
WHERE devFirstConnection IS NOT NULL
LIMIT 10
""")
samples = []
for row in sql.fetchall():
for ts in row:
if ts:
samples.append(ts)
if not samples:
mylog("verbose", "[db_upgrade] No timestamp samples found in Devices - assuming UTC")
return True # Empty DB, assume UTC
# Parse samples and check if they have timezone info (which would indicate migration already done)
has_tz_marker = any('+' in str(ts) or 'Z' in str(ts) for ts in samples)
if has_tz_marker:
mylog("verbose", "[db_upgrade] Timestamps have timezone markers - already migrated to UTC")
return True
mylog("debug", f"[db_upgrade] Sampled {len(samples)} timestamps. Current TZ offset: {current_offset_seconds}s")
mylog("verbose", "[db_upgrade] Timestamps appear to be in system local time - migration needed")
return False
except Exception as e:
mylog("warn", f"[db_upgrade] Error checking UTC status: {e} - assuming UTC")
return True
@@ -574,63 +570,91 @@ def is_timestamps_in_utc(sql) -> bool:
def migrate_timestamps_to_utc(sql) -> bool:
"""
Migrate all timestamp columns in the database from local time to UTC.
This function determines if migration is needed based on the VERSION setting:
- Fresh installs (no VERSION): Skip migration - timestamps already UTC from timeNowUTC()
- Version >= 26.2.6: Skip migration - already using UTC timestamps
- Version < 26.2.6: Run migration - convert local timestamps to UTC
Affected tables:
- Devices: devFirstConnection, devLastConnection, devLastNotification
- Events: eve_DateTime
- Sessions: ses_DateTimeConnection, ses_DateTimeDisconnection
- Notifications: DateTimeCreated, DateTimePushed
- Online_History: Scan_Date
- Plugins_Objects: DateTimeCreated, DateTimeChanged
- Plugins_Events: DateTimeCreated, DateTimeChanged
- Plugins_History: DateTimeCreated, DateTimeChanged
- AppEvents: DateTimeCreated
Safely migrate timestamp columns from local time to UTC.
Migration rules (fail-safe):
- Default behaviour: RUN migration unless proven safe to skip
- Version > 26.2.6 → timestamps already UTC → skip
- Missing / unknown / unparsable version → migrate
- Migration flag present → skip
- Detection says already UTC → skip
Returns:
bool: True if migration completed or wasn't needed, False on error
bool: True if migration completed or not needed, False on error
"""
try:
import conf
from zoneinfo import ZoneInfo
import datetime as dt
# Check VERSION from Settings table (from previous app run)
sql.execute("SELECT setValue FROM Settings WHERE setKey = 'VERSION'")
# -------------------------------------------------
# Check migration flag (idempotency protection)
# -------------------------------------------------
try:
sql.execute("SELECT setValue FROM Settings WHERE setKey='DB_TIMESTAMPS_UTC_MIGRATED'")
result = sql.fetchone()
if result and str(result[0]) == "1":
mylog("verbose", "[db_upgrade] UTC timestamp migration already completed - skipping")
return True
except Exception:
pass
# -------------------------------------------------
# Read previous version
# -------------------------------------------------
sql.execute("SELECT setValue FROM Settings WHERE setKey='VERSION'")
result = sql.fetchone()
prev_version = result[0] if result else ""
# Fresh install: VERSION is empty → timestamps already UTC from timeNowUTC()
if not prev_version or prev_version == "" or prev_version == "unknown":
mylog("verbose", "[db_upgrade] Fresh install detected - timestamps already in UTC format")
mylog("verbose", f"[db_upgrade] Version '{prev_version}' detected.")
# Default behaviour: migrate unless proven safe
should_migrate = True
# -------------------------------------------------
# Version-based safety check
# -------------------------------------------------
if prev_version and str(prev_version).lower() != "unknown":
try:
version_parts = prev_version.lstrip('v').split('.')
major = int(version_parts[0]) if len(version_parts) > 0 else 0
minor = int(version_parts[1]) if len(version_parts) > 1 else 0
patch = int(version_parts[2]) if len(version_parts) > 2 else 0
# UTC timestamps introduced AFTER v26.2.6
if (major, minor, patch) > (26, 2, 6):
should_migrate = False
mylog(
"verbose",
f"[db_upgrade] Version {prev_version} confirmed UTC timestamps - skipping migration",
)
except (ValueError, IndexError) as e:
mylog(
"warn",
f"[db_upgrade] Could not parse version '{prev_version}': {e} - running migration as safety measure",
)
else:
mylog(
"warn",
"[db_upgrade] VERSION missing/unknown - running migration as safety measure",
)
# -------------------------------------------------
# Detection fallback
# -------------------------------------------------
if should_migrate:
try:
if is_timestamps_in_utc(sql):
mylog(
"verbose",
"[db_upgrade] Timestamps appear already UTC - skipping migration",
)
return True
except Exception as e:
mylog(
"warn",
f"[db_upgrade] UTC detection failed ({e}) - continuing with migration",
)
else:
return True
# Parse version - format: "26.2.6" or "v26.2.6"
try:
version_parts = prev_version.strip('v').split('.')
major = int(version_parts[0]) if len(version_parts) > 0 else 0
minor = int(version_parts[1]) if len(version_parts) > 1 else 0
patch = int(version_parts[2]) if len(version_parts) > 2 else 0
# UTC timestamps introduced in v26.2.6
# If upgrading from 26.2.6 or later, timestamps are already UTC
if (major > 26) or (major == 26 and minor > 2) or (major == 26 and minor == 2 and patch >= 6):
mylog("verbose", f"[db_upgrade] Version {prev_version} already uses UTC timestamps - skipping migration")
return True
mylog("verbose", f"[db_upgrade] Upgrading from {prev_version} (< v26.2.6) - migrating timestamps to UTC")
except (ValueError, IndexError) as e:
mylog("warn", f"[db_upgrade] Could not parse version '{prev_version}': {e} - checking timestamps")
# Fallback: use detection logic
if is_timestamps_in_utc(sql):
mylog("verbose", "[db_upgrade] Timestamps appear to be in UTC - skipping migration")
return True
# Get timezone offset
try:
@@ -642,15 +666,15 @@ def migrate_timestamps_to_utc(sql) -> bool:
tz = None
except Exception:
tz = None
if tz:
now_local = dt.datetime.now(tz)
offset_hours = (now_local.utcoffset().total_seconds()) / 3600
else:
offset_hours = 0
mylog("verbose", f"[db_upgrade] Starting UTC timestamp migration (offset: {offset_hours} hours)")
# List of tables and their datetime columns
timestamp_columns = {
'Devices': ['devFirstConnection', 'devLastConnection', 'devLastNotification'],
@@ -663,7 +687,7 @@ def migrate_timestamps_to_utc(sql) -> bool:
'Plugins_History': ['DateTimeCreated', 'DateTimeChanged'],
'AppEvents': ['DateTimeCreated'],
}
for table, columns in timestamp_columns.items():
try:
# Check if table exists
@@ -671,7 +695,7 @@ def migrate_timestamps_to_utc(sql) -> bool:
if not sql.fetchone():
mylog("debug", f"[db_upgrade] Table '{table}' does not exist - skipping")
continue
for column in columns:
try:
# Update non-NULL timestamps
@@ -691,21 +715,21 @@ def migrate_timestamps_to_utc(sql) -> bool:
SET {column} = DATETIME({column}, '+{abs_hours} hours', '+{abs_mins} minutes')
WHERE {column} IS NOT NULL
""")
row_count = sql.rowcount
if row_count > 0:
mylog("verbose", f"[db_upgrade] Migrated {row_count} timestamps in {table}.{column}")
except Exception as e:
mylog("warn", f"[db_upgrade] Error updating {table}.{column}: {e}")
continue
except Exception as e:
mylog("warn", f"[db_upgrade] Error processing table {table}: {e}")
continue
mylog("none", "[db_upgrade] ✓ UTC timestamp migration completed successfully")
return True
except Exception as e:
mylog("none", f"[db_upgrade] ERROR during timestamp migration: {e}")
return False