vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2026-04-10 20:22:02 -07:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

BE+FE: refactor timezone UTC #1506

Browse Source 
Signed-off-by: jokob-sk <jokob.sk@gmail.com>
This commit is contained in:
jokob-sk
2026-02-11 16:15:49 +11:00
parent 70c3530a5c
commit b57d36607a
4 changed files with 116 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
front/deviceDetailsEvents.php
View File

@@ -116,7 +116,7 @@ function initializeEventsDatatable (eventsRows) {
{ {
targets: [0], targets: [0],
'createdCell': function (td, cellData, rowData, row, col) { 'createdCell': function (td, cellData, rowData, row, col) {
$(td).html(translateHTMLcodes(localizeTimestamp(cellData))); $(td).html(translateHTMLcodes((cellData)));
} }
} }
], ],

1
front/js/common.js
View File

@@ -447,6 +447,7 @@ function localizeTimestamp(input) {
return formatSafe(input, tz); return formatSafe(input, tz);
function formatSafe(str, tz) { function formatSafe(str, tz) {
// CHECK: Does the input string have timezone information? // CHECK: Does the input string have timezone information?
// - Ends with Z: "2026-02-11T11:37:02Z" // - Ends with Z: "2026-02-11T11:37:02Z"
// - Has GMT±offset: "Wed Feb 11 2026 12:34:12 GMT+1100 (...)" // - Has GMT±offset: "Wed Feb 11 2026 12:34:12 GMT+1100 (...)"

180
server/db/db_upgrade.py
View File

@@ -1,10 +1,6 @@
import sys import conf
import os from zoneinfo import ZoneInfo
import datetime as dt
# Register NetAlertX directories
INSTALL_PATH = os.getenv("NETALERTX_APP", "/app")
sys.path.extend([f"{INSTALL_PATH}/server"])
from logger import mylog # noqa: E402 [flake8 lint suppression] from logger import mylog # noqa: E402 [flake8 lint suppression]
from messaging.in_app import write_notification # noqa: E402 [flake8 lint suppression] from messaging.in_app import write_notification # noqa: E402 [flake8 lint suppression]
@@ -503,14 +499,14 @@ def ensure_plugins_tables(sql) -> bool:
def is_timestamps_in_utc(sql) -> bool: def is_timestamps_in_utc(sql) -> bool:
""" """
Check if existing timestamps in Devices table are already in UTC format. Check if existing timestamps in Devices table are already in UTC format.
Strategy: Strategy:
1. Sample 10 non-NULL devFirstConnection timestamps from Devices 1. Sample 10 non-NULL devFirstConnection timestamps from Devices
2. For each timestamp, assume it's UTC and calculate what it would be in local time 2. For each timestamp, assume it's UTC and calculate what it would be in local time
3. Check if timestamps have a consistent offset pattern (indicating local time storage) 3. Check if timestamps have a consistent offset pattern (indicating local time storage)
4. If offset is consistently > 0, they're likely local timestamps (need migration) 4. If offset is consistently > 0, they're likely local timestamps (need migration)
5. If offset is ~0 or inconsistent, they're likely already UTC (skip migration) 5. If offset is ~0 or inconsistent, they're likely already UTC (skip migration)
Returns: Returns:
bool: True if timestamps appear to be in UTC already, False if they need migration bool: True if timestamps appear to be in UTC already, False if they need migration
""" """
@@ -519,10 +515,10 @@ def is_timestamps_in_utc(sql) -> bool:
import conf import conf
from zoneinfo import ZoneInfo from zoneinfo import ZoneInfo
import datetime as dt import datetime as dt
now = dt.datetime.now(dt.UTC).replace(microsecond=0) now = dt.datetime.now(dt.UTC).replace(microsecond=0)
current_offset_seconds = 0 current_offset_seconds = 0
try: try:
if isinstance(conf.tz, dt.tzinfo): if isinstance(conf.tz, dt.tzinfo):
tz = conf.tz tz = conf.tz
@@ -532,13 +528,13 @@ def is_timestamps_in_utc(sql) -> bool:
tz = None tz = None
except Exception: except Exception:
tz = None tz = None
if tz: if tz:
local_now = dt.datetime.now(tz).replace(microsecond=0) local_now = dt.datetime.now(tz).replace(microsecond=0)
local_offset = local_now.utcoffset().total_seconds() local_offset = local_now.utcoffset().total_seconds()
utc_offset = now.utcoffset().total_seconds() if now.utcoffset() else 0 utc_offset = now.utcoffset().total_seconds() if now.utcoffset() else 0
current_offset_seconds = int(local_offset - utc_offset) current_offset_seconds = int(local_offset - utc_offset)
# Sample timestamps from Devices table # Sample timestamps from Devices table
sql.execute(""" sql.execute("""
SELECT devFirstConnection, devLastConnection, devLastNotification SELECT devFirstConnection, devLastConnection, devLastNotification
@@ -546,27 +542,27 @@ def is_timestamps_in_utc(sql) -> bool:
WHERE devFirstConnection IS NOT NULL WHERE devFirstConnection IS NOT NULL
LIMIT 10 LIMIT 10
""") """)
samples = [] samples = []
for row in sql.fetchall(): for row in sql.fetchall():
for ts in row: for ts in row:
if ts: if ts:
samples.append(ts) samples.append(ts)
if not samples: if not samples:
mylog("verbose", "[db_upgrade] No timestamp samples found in Devices - assuming UTC") mylog("verbose", "[db_upgrade] No timestamp samples found in Devices - assuming UTC")
return True # Empty DB, assume UTC return True # Empty DB, assume UTC
# Parse samples and check if they have timezone info (which would indicate migration already done) # Parse samples and check if they have timezone info (which would indicate migration already done)
has_tz_marker = any('+' in str(ts) or 'Z' in str(ts) for ts in samples) has_tz_marker = any('+' in str(ts) or 'Z' in str(ts) for ts in samples)
if has_tz_marker: if has_tz_marker:
mylog("verbose", "[db_upgrade] Timestamps have timezone markers - already migrated to UTC") mylog("verbose", "[db_upgrade] Timestamps have timezone markers - already migrated to UTC")
return True return True
mylog("debug", f"[db_upgrade] Sampled {len(samples)} timestamps. Current TZ offset: {current_offset_seconds}s") mylog("debug", f"[db_upgrade] Sampled {len(samples)} timestamps. Current TZ offset: {current_offset_seconds}s")
mylog("verbose", "[db_upgrade] Timestamps appear to be in system local time - migration needed") mylog("verbose", "[db_upgrade] Timestamps appear to be in system local time - migration needed")
return False return False
except Exception as e: except Exception as e:
mylog("warn", f"[db_upgrade] Error checking UTC status: {e} - assuming UTC") mylog("warn", f"[db_upgrade] Error checking UTC status: {e} - assuming UTC")
return True return True
@@ -574,63 +570,91 @@ def is_timestamps_in_utc(sql) -> bool:
def migrate_timestamps_to_utc(sql) -> bool: def migrate_timestamps_to_utc(sql) -> bool:
""" """
Migrate all timestamp columns in the database from local time to UTC. Safely migrate timestamp columns from local time to UTC.
This function determines if migration is needed based on the VERSION setting: Migration rules (fail-safe):
- Fresh installs (no VERSION): Skip migration - timestamps already UTC from timeNowUTC() - Default behaviour: RUN migration unless proven safe to skip
- Version >= 26.2.6: Skip migration - already using UTC timestamps - Version > 26.2.6 → timestamps already UTC → skip
- Version < 26.2.6: Run migration - convert local timestamps to UTC - Missing / unknown / unparsable version → migrate
- Migration flag present → skip
Affected tables: - Detection says already UTC → skip
- Devices: devFirstConnection, devLastConnection, devLastNotification
- Events: eve_DateTime
- Sessions: ses_DateTimeConnection, ses_DateTimeDisconnection
- Notifications: DateTimeCreated, DateTimePushed
- Online_History: Scan_Date
- Plugins_Objects: DateTimeCreated, DateTimeChanged
- Plugins_Events: DateTimeCreated, DateTimeChanged
- Plugins_History: DateTimeCreated, DateTimeChanged
- AppEvents: DateTimeCreated
Returns: Returns:
bool: True if migration completed or wasn't needed, False on error bool: True if migration completed or not needed, False on error
""" """
try: try:
import conf # -------------------------------------------------
from zoneinfo import ZoneInfo # Check migration flag (idempotency protection)
import datetime as dt # -------------------------------------------------
try:
# Check VERSION from Settings table (from previous app run) sql.execute("SELECT setValue FROM Settings WHERE setKey='DB_TIMESTAMPS_UTC_MIGRATED'")
sql.execute("SELECT setValue FROM Settings WHERE setKey = 'VERSION'") result = sql.fetchone()
if result and str(result[0]) == "1":
mylog("verbose", "[db_upgrade] UTC timestamp migration already completed - skipping")
return True
except Exception:
pass
# -------------------------------------------------
# Read previous version
# -------------------------------------------------
sql.execute("SELECT setValue FROM Settings WHERE setKey='VERSION'")
result = sql.fetchone() result = sql.fetchone()
prev_version = result[0] if result else "" prev_version = result[0] if result else ""
# Fresh install: VERSION is empty → timestamps already UTC from timeNowUTC() mylog("verbose", f"[db_upgrade] Version '{prev_version}' detected.")
if not prev_version or prev_version == "" or prev_version == "unknown":
mylog("verbose", "[db_upgrade] Fresh install detected - timestamps already in UTC format") # Default behaviour: migrate unless proven safe
should_migrate = True
# -------------------------------------------------
# Version-based safety check
# -------------------------------------------------
if prev_version and str(prev_version).lower() != "unknown":
try:
version_parts = prev_version.lstrip('v').split('.')
major = int(version_parts[0]) if len(version_parts) > 0 else 0
minor = int(version_parts[1]) if len(version_parts) > 1 else 0
patch = int(version_parts[2]) if len(version_parts) > 2 else 0
# UTC timestamps introduced AFTER v26.2.6
if (major, minor, patch) > (26, 2, 6):
should_migrate = False
mylog(
"verbose",
f"[db_upgrade] Version {prev_version} confirmed UTC timestamps - skipping migration",
)
except (ValueError, IndexError) as e:
mylog(
"warn",
f"[db_upgrade] Could not parse version '{prev_version}': {e} - running migration as safety measure",
)
else:
mylog(
"warn",
"[db_upgrade] VERSION missing/unknown - running migration as safety measure",
)
# -------------------------------------------------
# Detection fallback
# -------------------------------------------------
if should_migrate:
try:
if is_timestamps_in_utc(sql):
mylog(
"verbose",
"[db_upgrade] Timestamps appear already UTC - skipping migration",
)
return True
except Exception as e:
mylog(
"warn",
f"[db_upgrade] UTC detection failed ({e}) - continuing with migration",
)
else:
return True return True
# Parse version - format: "26.2.6" or "v26.2.6"
try:
version_parts = prev_version.strip('v').split('.')
major = int(version_parts[0]) if len(version_parts) > 0 else 0
minor = int(version_parts[1]) if len(version_parts) > 1 else 0
patch = int(version_parts[2]) if len(version_parts) > 2 else 0
# UTC timestamps introduced in v26.2.6
# If upgrading from 26.2.6 or later, timestamps are already UTC
if (major > 26) or (major == 26 and minor > 2) or (major == 26 and minor == 2 and patch >= 6):
mylog("verbose", f"[db_upgrade] Version {prev_version} already uses UTC timestamps - skipping migration")
return True
mylog("verbose", f"[db_upgrade] Upgrading from {prev_version} (< v26.2.6) - migrating timestamps to UTC")
except (ValueError, IndexError) as e:
mylog("warn", f"[db_upgrade] Could not parse version '{prev_version}': {e} - checking timestamps")
# Fallback: use detection logic
if is_timestamps_in_utc(sql):
mylog("verbose", "[db_upgrade] Timestamps appear to be in UTC - skipping migration")
return True
# Get timezone offset # Get timezone offset
try: try:
@@ -642,15 +666,15 @@ def migrate_timestamps_to_utc(sql) -> bool:
tz = None tz = None
except Exception: except Exception:
tz = None tz = None
if tz: if tz:
now_local = dt.datetime.now(tz) now_local = dt.datetime.now(tz)
offset_hours = (now_local.utcoffset().total_seconds()) / 3600 offset_hours = (now_local.utcoffset().total_seconds()) / 3600
else: else:
offset_hours = 0 offset_hours = 0
mylog("verbose", f"[db_upgrade] Starting UTC timestamp migration (offset: {offset_hours} hours)") mylog("verbose", f"[db_upgrade] Starting UTC timestamp migration (offset: {offset_hours} hours)")
# List of tables and their datetime columns # List of tables and their datetime columns
timestamp_columns = { timestamp_columns = {
'Devices': ['devFirstConnection', 'devLastConnection', 'devLastNotification'], 'Devices': ['devFirstConnection', 'devLastConnection', 'devLastNotification'],
@@ -663,7 +687,7 @@ def migrate_timestamps_to_utc(sql) -> bool:
'Plugins_History': ['DateTimeCreated', 'DateTimeChanged'], 'Plugins_History': ['DateTimeCreated', 'DateTimeChanged'],
'AppEvents': ['DateTimeCreated'], 'AppEvents': ['DateTimeCreated'],
} }
for table, columns in timestamp_columns.items(): for table, columns in timestamp_columns.items():
try: try:
# Check if table exists # Check if table exists
@@ -671,7 +695,7 @@ def migrate_timestamps_to_utc(sql) -> bool:
if not sql.fetchone(): if not sql.fetchone():
mylog("debug", f"[db_upgrade] Table '{table}' does not exist - skipping") mylog("debug", f"[db_upgrade] Table '{table}' does not exist - skipping")
continue continue
for column in columns: for column in columns:
try: try:
# Update non-NULL timestamps # Update non-NULL timestamps
@@ -691,21 +715,21 @@ def migrate_timestamps_to_utc(sql) -> bool:
SET {column} = DATETIME({column}, '+{abs_hours} hours', '+{abs_mins} minutes') SET {column} = DATETIME({column}, '+{abs_hours} hours', '+{abs_mins} minutes')
WHERE {column} IS NOT NULL WHERE {column} IS NOT NULL
""") """)
row_count = sql.rowcount row_count = sql.rowcount
if row_count > 0: if row_count > 0:
mylog("verbose", f"[db_upgrade] Migrated {row_count} timestamps in {table}.{column}") mylog("verbose", f"[db_upgrade] Migrated {row_count} timestamps in {table}.{column}")
except Exception as e: except Exception as e:
mylog("warn", f"[db_upgrade] Error updating {table}.{column}: {e}") mylog("warn", f"[db_upgrade] Error updating {table}.{column}: {e}")
continue continue
except Exception as e: except Exception as e:
mylog("warn", f"[db_upgrade] Error processing table {table}: {e}") mylog("warn", f"[db_upgrade] Error processing table {table}: {e}")
continue continue
mylog("none", "[db_upgrade] ✓ UTC timestamp migration completed successfully") mylog("none", "[db_upgrade] ✓ UTC timestamp migration completed successfully")
return True return True
except Exception as e: except Exception as e:
mylog("none", f"[db_upgrade] ERROR during timestamp migration: {e}") mylog("none", f"[db_upgrade] ERROR during timestamp migration: {e}")
return False return False

12
server/models/plugin_object_instance.py
View File

@@ -49,6 +49,18 @@ class PluginObjectInstance:
"SELECT * FROM Plugins_Objects WHERE Plugin = ?", (plugin,) "SELECT * FROM Plugins_Objects WHERE Plugin = ?", (plugin,)
) )
def getLastNCreatedPerPLugin(self, plugin, entries=1):
return self._fetchall(
"""
SELECT *
FROM Plugins_Objects
WHERE Plugin = ?
ORDER BY DateTimeCreated DESC
LIMIT ?
""",
(plugin, entries),
)
def getByField(self, plugPrefix, matchedColumn, matchedKey, returnFields=None): def getByField(self, plugPrefix, matchedColumn, matchedKey, returnFields=None):
rows = self._fetchall( rows = self._fetchall(
f"SELECT * FROM Plugins_Objects WHERE Plugin = ? AND {matchedColumn} = ?", f"SELECT * FROM Plugins_Objects WHERE Plugin = ? AND {matchedColumn} = ?",