diff --git a/front/deviceDetailsEvents.php b/front/deviceDetailsEvents.php index b11e4b52..a592d8ce 100755 --- a/front/deviceDetailsEvents.php +++ b/front/deviceDetailsEvents.php @@ -116,7 +116,7 @@ function initializeEventsDatatable (eventsRows) { { targets: [0], 'createdCell': function (td, cellData, rowData, row, col) { - $(td).html(translateHTMLcodes(localizeTimestamp(cellData))); + $(td).html(translateHTMLcodes((cellData))); } } ], diff --git a/front/js/common.js b/front/js/common.js index 4753db38..212d67fa 100755 --- a/front/js/common.js +++ b/front/js/common.js @@ -447,6 +447,7 @@ function localizeTimestamp(input) { return formatSafe(input, tz); function formatSafe(str, tz) { + // CHECK: Does the input string have timezone information? // - Ends with Z: "2026-02-11T11:37:02Z" // - Has GMT±offset: "Wed Feb 11 2026 12:34:12 GMT+1100 (...)" diff --git a/server/db/db_upgrade.py b/server/db/db_upgrade.py index 13197f39..919fb913 100755 --- a/server/db/db_upgrade.py +++ b/server/db/db_upgrade.py @@ -1,10 +1,6 @@ -import sys -import os - -# Register NetAlertX directories -INSTALL_PATH = os.getenv("NETALERTX_APP", "/app") -sys.path.extend([f"{INSTALL_PATH}/server"]) - +import conf +from zoneinfo import ZoneInfo +import datetime as dt from logger import mylog # noqa: E402 [flake8 lint suppression] from messaging.in_app import write_notification # noqa: E402 [flake8 lint suppression] @@ -503,14 +499,14 @@ def ensure_plugins_tables(sql) -> bool: def is_timestamps_in_utc(sql) -> bool: """ Check if existing timestamps in Devices table are already in UTC format. - + Strategy: 1. Sample 10 non-NULL devFirstConnection timestamps from Devices 2. For each timestamp, assume it's UTC and calculate what it would be in local time 3. Check if timestamps have a consistent offset pattern (indicating local time storage) 4. If offset is consistently > 0, they're likely local timestamps (need migration) 5. If offset is ~0 or inconsistent, they're likely already UTC (skip migration) - + Returns: bool: True if timestamps appear to be in UTC already, False if they need migration """ @@ -519,10 +515,10 @@ def is_timestamps_in_utc(sql) -> bool: import conf from zoneinfo import ZoneInfo import datetime as dt - + now = dt.datetime.now(dt.UTC).replace(microsecond=0) current_offset_seconds = 0 - + try: if isinstance(conf.tz, dt.tzinfo): tz = conf.tz @@ -532,13 +528,13 @@ def is_timestamps_in_utc(sql) -> bool: tz = None except Exception: tz = None - + if tz: local_now = dt.datetime.now(tz).replace(microsecond=0) local_offset = local_now.utcoffset().total_seconds() utc_offset = now.utcoffset().total_seconds() if now.utcoffset() else 0 current_offset_seconds = int(local_offset - utc_offset) - + # Sample timestamps from Devices table sql.execute(""" SELECT devFirstConnection, devLastConnection, devLastNotification @@ -546,27 +542,27 @@ def is_timestamps_in_utc(sql) -> bool: WHERE devFirstConnection IS NOT NULL LIMIT 10 """) - + samples = [] for row in sql.fetchall(): for ts in row: if ts: samples.append(ts) - + if not samples: mylog("verbose", "[db_upgrade] No timestamp samples found in Devices - assuming UTC") return True # Empty DB, assume UTC - + # Parse samples and check if they have timezone info (which would indicate migration already done) has_tz_marker = any('+' in str(ts) or 'Z' in str(ts) for ts in samples) if has_tz_marker: mylog("verbose", "[db_upgrade] Timestamps have timezone markers - already migrated to UTC") return True - + mylog("debug", f"[db_upgrade] Sampled {len(samples)} timestamps. Current TZ offset: {current_offset_seconds}s") mylog("verbose", "[db_upgrade] Timestamps appear to be in system local time - migration needed") return False - + except Exception as e: mylog("warn", f"[db_upgrade] Error checking UTC status: {e} - assuming UTC") return True @@ -574,63 +570,91 @@ def is_timestamps_in_utc(sql) -> bool: def migrate_timestamps_to_utc(sql) -> bool: """ - Migrate all timestamp columns in the database from local time to UTC. - - This function determines if migration is needed based on the VERSION setting: - - Fresh installs (no VERSION): Skip migration - timestamps already UTC from timeNowUTC() - - Version >= 26.2.6: Skip migration - already using UTC timestamps - - Version < 26.2.6: Run migration - convert local timestamps to UTC - - Affected tables: - - Devices: devFirstConnection, devLastConnection, devLastNotification - - Events: eve_DateTime - - Sessions: ses_DateTimeConnection, ses_DateTimeDisconnection - - Notifications: DateTimeCreated, DateTimePushed - - Online_History: Scan_Date - - Plugins_Objects: DateTimeCreated, DateTimeChanged - - Plugins_Events: DateTimeCreated, DateTimeChanged - - Plugins_History: DateTimeCreated, DateTimeChanged - - AppEvents: DateTimeCreated - + Safely migrate timestamp columns from local time to UTC. + + Migration rules (fail-safe): + - Default behaviour: RUN migration unless proven safe to skip + - Version > 26.2.6 → timestamps already UTC → skip + - Missing / unknown / unparsable version → migrate + - Migration flag present → skip + - Detection says already UTC → skip + Returns: - bool: True if migration completed or wasn't needed, False on error + bool: True if migration completed or not needed, False on error """ + try: - import conf - from zoneinfo import ZoneInfo - import datetime as dt - - # Check VERSION from Settings table (from previous app run) - sql.execute("SELECT setValue FROM Settings WHERE setKey = 'VERSION'") + # ------------------------------------------------- + # Check migration flag (idempotency protection) + # ------------------------------------------------- + try: + sql.execute("SELECT setValue FROM Settings WHERE setKey='DB_TIMESTAMPS_UTC_MIGRATED'") + result = sql.fetchone() + if result and str(result[0]) == "1": + mylog("verbose", "[db_upgrade] UTC timestamp migration already completed - skipping") + return True + except Exception: + pass + + # ------------------------------------------------- + # Read previous version + # ------------------------------------------------- + sql.execute("SELECT setValue FROM Settings WHERE setKey='VERSION'") result = sql.fetchone() prev_version = result[0] if result else "" - - # Fresh install: VERSION is empty → timestamps already UTC from timeNowUTC() - if not prev_version or prev_version == "" or prev_version == "unknown": - mylog("verbose", "[db_upgrade] Fresh install detected - timestamps already in UTC format") + + mylog("verbose", f"[db_upgrade] Version '{prev_version}' detected.") + + # Default behaviour: migrate unless proven safe + should_migrate = True + + # ------------------------------------------------- + # Version-based safety check + # ------------------------------------------------- + if prev_version and str(prev_version).lower() != "unknown": + try: + version_parts = prev_version.lstrip('v').split('.') + major = int(version_parts[0]) if len(version_parts) > 0 else 0 + minor = int(version_parts[1]) if len(version_parts) > 1 else 0 + patch = int(version_parts[2]) if len(version_parts) > 2 else 0 + + # UTC timestamps introduced AFTER v26.2.6 + if (major, minor, patch) > (26, 2, 6): + should_migrate = False + mylog( + "verbose", + f"[db_upgrade] Version {prev_version} confirmed UTC timestamps - skipping migration", + ) + + except (ValueError, IndexError) as e: + mylog( + "warn", + f"[db_upgrade] Could not parse version '{prev_version}': {e} - running migration as safety measure", + ) + else: + mylog( + "warn", + "[db_upgrade] VERSION missing/unknown - running migration as safety measure", + ) + + # ------------------------------------------------- + # Detection fallback + # ------------------------------------------------- + if should_migrate: + try: + if is_timestamps_in_utc(sql): + mylog( + "verbose", + "[db_upgrade] Timestamps appear already UTC - skipping migration", + ) + return True + except Exception as e: + mylog( + "warn", + f"[db_upgrade] UTC detection failed ({e}) - continuing with migration", + ) + else: return True - - # Parse version - format: "26.2.6" or "v26.2.6" - try: - version_parts = prev_version.strip('v').split('.') - major = int(version_parts[0]) if len(version_parts) > 0 else 0 - minor = int(version_parts[1]) if len(version_parts) > 1 else 0 - patch = int(version_parts[2]) if len(version_parts) > 2 else 0 - - # UTC timestamps introduced in v26.2.6 - # If upgrading from 26.2.6 or later, timestamps are already UTC - if (major > 26) or (major == 26 and minor > 2) or (major == 26 and minor == 2 and patch >= 6): - mylog("verbose", f"[db_upgrade] Version {prev_version} already uses UTC timestamps - skipping migration") - return True - - mylog("verbose", f"[db_upgrade] Upgrading from {prev_version} (< v26.2.6) - migrating timestamps to UTC") - - except (ValueError, IndexError) as e: - mylog("warn", f"[db_upgrade] Could not parse version '{prev_version}': {e} - checking timestamps") - # Fallback: use detection logic - if is_timestamps_in_utc(sql): - mylog("verbose", "[db_upgrade] Timestamps appear to be in UTC - skipping migration") - return True # Get timezone offset try: @@ -642,15 +666,15 @@ def migrate_timestamps_to_utc(sql) -> bool: tz = None except Exception: tz = None - + if tz: now_local = dt.datetime.now(tz) offset_hours = (now_local.utcoffset().total_seconds()) / 3600 else: offset_hours = 0 - + mylog("verbose", f"[db_upgrade] Starting UTC timestamp migration (offset: {offset_hours} hours)") - + # List of tables and their datetime columns timestamp_columns = { 'Devices': ['devFirstConnection', 'devLastConnection', 'devLastNotification'], @@ -663,7 +687,7 @@ def migrate_timestamps_to_utc(sql) -> bool: 'Plugins_History': ['DateTimeCreated', 'DateTimeChanged'], 'AppEvents': ['DateTimeCreated'], } - + for table, columns in timestamp_columns.items(): try: # Check if table exists @@ -671,7 +695,7 @@ def migrate_timestamps_to_utc(sql) -> bool: if not sql.fetchone(): mylog("debug", f"[db_upgrade] Table '{table}' does not exist - skipping") continue - + for column in columns: try: # Update non-NULL timestamps @@ -691,21 +715,21 @@ def migrate_timestamps_to_utc(sql) -> bool: SET {column} = DATETIME({column}, '+{abs_hours} hours', '+{abs_mins} minutes') WHERE {column} IS NOT NULL """) - + row_count = sql.rowcount if row_count > 0: mylog("verbose", f"[db_upgrade] Migrated {row_count} timestamps in {table}.{column}") except Exception as e: mylog("warn", f"[db_upgrade] Error updating {table}.{column}: {e}") continue - + except Exception as e: mylog("warn", f"[db_upgrade] Error processing table {table}: {e}") continue - + mylog("none", "[db_upgrade] ✓ UTC timestamp migration completed successfully") return True - + except Exception as e: mylog("none", f"[db_upgrade] ERROR during timestamp migration: {e}") return False diff --git a/server/models/plugin_object_instance.py b/server/models/plugin_object_instance.py index 3d4ceaf2..1f13d69c 100755 --- a/server/models/plugin_object_instance.py +++ b/server/models/plugin_object_instance.py @@ -49,6 +49,18 @@ class PluginObjectInstance: "SELECT * FROM Plugins_Objects WHERE Plugin = ?", (plugin,) ) + def getLastNCreatedPerPLugin(self, plugin, entries=1): + return self._fetchall( + """ + SELECT * + FROM Plugins_Objects + WHERE Plugin = ? + ORDER BY DateTimeCreated DESC + LIMIT ? + """, + (plugin, entries), + ) + def getByField(self, plugPrefix, matchedColumn, matchedKey, returnFields=None): rows = self._fetchall( f"SELECT * FROM Plugins_Objects WHERE Plugin = ? AND {matchedColumn} = ?",