From 170a3c0ae143332eaedf8dbad5781ad1700585da Mon Sep 17 00:00:00 2001
From: jokob-sk <jokob-sk@gmail.com>
Date: Sat, 26 Jul 2025 08:45:07 +1000
Subject: [PATCH] presence rework #1119, plugin history filter

---
 front/deviceDetails.php        |  28 ++++----
 front/deviceDetailsEvents.php  |  65 ++++++++++++++----
 front/js/common.js             |  72 ++++++++++++++++----
 front/php/server/events.php    |  38 -----------
 front/pluginsCore.php          |   8 +--
 server/scan/device_handling.py |  50 +++++++++-----
 server/scan/session_events.py  | 121 +++++++--------------------------
 7 files changed, 186 insertions(+), 196 deletions(-)

diff --git a/front/deviceDetails.php b/front/deviceDetails.php
index e46f9281..8411e69c 100755
--- a/front/deviceDetails.php
+++ b/front/deviceDetails.php
@@ -286,21 +286,21 @@ function main () {
 
   
 
-  // Events tab toggle conenction events
-  $('input').on('ifToggled', function(event){
-    // Hide / Show Events
-    if (event.currentTarget.id == 'chkHideConnectionEvents') {
-      getDeviceEvents();
-    } else {
-      // Activate save & restore
-      // activateSaveRestoreData();
+  // // Events tab toggle conenction events
+  // $('input').on('ifToggled', function(event){
+  //   // Hide / Show Events
+  //   if (event.currentTarget.id == 'chkHideConnectionEvents') {
+  //     getDeviceEvents();
+  //   } else {
+  //     // Activate save & restore
+  //     // activateSaveRestoreData();
 
-      // Ask skip notifications
-      // if (event.currentTarget.id == 'chkArchived' ) {
-      //   askSkipNotifications();
-      // }
-    }
-  });
+  //     // Ask skip notifications
+  //     // if (event.currentTarget.id == 'chkArchived' ) {
+  //     //   askSkipNotifications();
+  //     // }
+  //   }
+  // });
        
 }
 
diff --git a/front/deviceDetailsEvents.php b/front/deviceDetailsEvents.php
index 401c7641..6e6d7f82 100755
--- a/front/deviceDetailsEvents.php
+++ b/front/deviceDetailsEvents.php
@@ -7,11 +7,11 @@
  
 
 <!-- Hide Connections -->
-<div class="text-center">
-    <label>
-    <input class="checkbox blue hidden" id="chkHideConnectionEvents" type="checkbox" checked>
-    <?= lang('DevDetail_Events_CheckBox');?>
+<div class="col-sm-12 col-xs-12">
+    <label class="col-sm-3 col-xs-10">      
+      <?= lang('DevDetail_Events_CheckBox');?>
     </label>
+    <input class="checkbox blue col-sm-1 col-xs-2" id="chkHideConnectionEvents" type="checkbox" onChange="loadEventsData()">
 </div>
 
 <!-- Datatable Events -->
@@ -29,21 +29,58 @@
 
 <script>
 
-  var eventsRows          = 10;
-  var eventsHide          = true;
-  var parEventsRows       = 'Front_Details_Events_Rows';
-  var parEventsHide       = 'Front_Details_Events_Hide';
+var eventsRows          = 10;
+var eventsHide          = true;
+var parEventsRows       = 'Front_Details_Events_Rows';
+var parEventsHide       = 'Front_Details_Events_Hide';
 
 
 
-  // -----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
+function loadEventsData() {
+  const hideConnections = $('#chkHideConnectionEvents')[0].checked;
+  const hideConnectionsStr = hideConnections ? 'true' : 'false';
+
+  const rawSql = `
+    SELECT eve_DateTime, eve_EventType, eve_IP, eve_AdditionalInfo
+    FROM Events 
+    WHERE eve_MAC = "${mac}" 
+      AND (
+        (eve_EventType NOT IN ("Connected", "Disconnected", "VOIDED - Connected", "VOIDED - Disconnected"))
+        OR "${hideConnectionsStr}" = "false"
+      )
+  `;
+
+  const apiUrl = `php/server/dbHelper.php?action=read&rawSql=${btoa(encodeURIComponent(rawSql))}`;
+
+  // Manually load the data first
+  $.get(apiUrl, function (data) {
+    const parsed = JSON.parse(data);
+
+    console.log(parsed);
+    
+    const rows = parsed.map(row => {
+      const rawDate = row.eve_DateTime;
+      const formattedDate = rawDate ? localizeTimestamp(rawDate) : '-';
+      return [
+        formattedDate,
+        row.eve_EventType,
+        row.eve_IP,
+        row.eve_AdditionalInfo
+      ];
+    });
+
+    console.log(rows);
+    
 
 
-  function loadEventsData() {
-    // Define Events datasource and query dada
-    hideConnections = $('#chkHideConnectionEvents')[0].checked;
-    $('#tableEvents').DataTable().ajax.url('php/server/events.php?action=getDeviceEvents&mac=' + mac +'&period='+ period +'&hideConnections='+ hideConnections).load();
-  }
+    // Fill the table manually
+    const table = $('#tableEvents').DataTable();
+    table.clear();
+    table.rows.add(rows); // assuming each row is an array
+    table.draw();
+  });
+}
 
 function initializeSessionsDatatable () {
 
diff --git a/front/js/common.js b/front/js/common.js
index f0b03169..685b9e8a 100755
--- a/front/js/common.js
+++ b/front/js/common.js
@@ -362,18 +362,64 @@ function getLangCode() {
 // -----------------------------------------------------------------------------
 // String utilities
 // -----------------------------------------------------------------------------
+function localizeTimestamp(input) {
+  let tz = getSetting("TIMEZONE") || 'Europe/Berlin';
 
+  // Convert to string and trim
+  input = String(input || '').trim();
 
-function localizeTimestamp(result)
-{
-  // contains TZ in format Europe/Berlin
-  tz = getSetting("TIMEZONE")
+  // Normalize multiple spaces and remove commas
+  const cleaned = input.replace(',', ' ').replace(/\s+/g, ' ');
 
-  // set default if not available or app still loading
-  tz == ""  ? tz = 'Europe/Berlin' : tz = tz;
-  
-  const date = new Date(result); // Assumes result is a timestamp or ISO string
-  const formatter = new Intl.DateTimeFormat('default', {
+  // DD/MM/YYYY format check
+  const dateTimeParts = cleaned.split(' ');
+  if (dateTimeParts.length >= 2 && dateTimeParts[0].includes('/')) {
+    const [day, month, year] = dateTimeParts[0].split('/');
+    const timePart = dateTimeParts[1];
+
+    if (day && month && year && timePart) {
+      const isoString = `${year}-${month}-${day}T${timePart.length === 5 ? timePart + ':00' : timePart}`;
+      const date = new Date(isoString);
+      if (!isFinite(date)) return 'b-';
+
+      return new Intl.DateTimeFormat('default', {
+        timeZone: tz,
+        year: 'numeric',
+        month: '2-digit',
+        day: '2-digit',
+        hour: '2-digit',
+        minute: '2-digit',
+        second: '2-digit',
+        hour12: false
+      }).format(date);
+    }
+  }
+
+  // ISO style YYYY-MM-DD HH:mm(:ss)?
+  const match = cleaned.match(/^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2})(:\d{2})?$/);
+  if (match) {
+    let iso = `${match[1]}T${match[2]}${match[3] || ':00'}`;
+
+    const date = new Date(iso);
+    if (!isFinite(date)) return 'c-';
+
+    return new Intl.DateTimeFormat('default', {
+      timeZone: tz,
+      year: 'numeric',
+      month: '2-digit',
+      day: '2-digit',
+      hour: '2-digit',
+      minute: '2-digit',
+      second: '2-digit',
+      hour12: false
+    }).format(date);
+  }
+
+  // Fallback: try to parse any other string input
+  const date = new Date(input);
+  if (!isFinite(date)) return 'Failed conversion: ' + input;
+
+  return new Intl.DateTimeFormat('default', {
     timeZone: tz,
     year: 'numeric',
     month: '2-digit',
@@ -381,12 +427,12 @@ function localizeTimestamp(result)
     hour: '2-digit',
     minute: '2-digit',
     second: '2-digit',
-    hour12: false // change to true if you want AM/PM format
-  });
-
-  return formatter.format(date);
+    hour12: false
+  }).format(date);
 }
 
+
+
 // ----------------------------------------------------
 /**
  * Replaces double quotes within single-quoted strings, then converts all single quotes to double quotes,
diff --git a/front/php/server/events.php b/front/php/server/events.php
index 3998718c..1fb73bc2 100755
--- a/front/php/server/events.php
+++ b/front/php/server/events.php
@@ -29,7 +29,6 @@ require_once  $_SERVER['DOCUMENT_ROOT'] . '/php/templates/security.php';
       case 'getEvents':          getEvents();                             break;                      
       case 'getDeviceSessions':  getDeviceSessions();                     break;
       case 'getDevicePresence':  getDevicePresence();                     break;
-      case 'getDeviceEvents':    getDeviceEvents();                       break;
       case 'getEventsCalendar':  getEventsCalendar();                     break;
       default:                   logServerConsole ('Action: '. $action);  break;
     }
@@ -410,41 +409,4 @@ function getEventsCalendar() {
   echo (json_encode($tableData));
 }
 
-
-//------------------------------------------------------------------------------
-//  Query Device events
-//------------------------------------------------------------------------------
-function getDeviceEvents() {
-  global $db;
-
-  // Request Parameters
-  $mac             = $_REQUEST['mac'];
-  $periodDate      = getDateFromPeriod();
-  $hideConnections = $_REQUEST ['hideConnections'];
-
-  // SQL 
-  $SQL = 'SELECT eve_DateTime, eve_EventType, eve_IP, eve_AdditionalInfo
-          FROM Events 
-          WHERE eve_MAC="'. $mac .'" AND eve_DateTime >= '. $periodDate .'
-            AND ( (eve_EventType <> "Connected" AND eve_EventType <> "Disconnected" AND
-                   eve_EventType <> "VOIDED - Connected" AND eve_EventType <> "VOIDED - Disconnected")
-                 OR "'. $hideConnections .'" = "false" ) ';
-  $result = $db->query($SQL);
-
-  // arrays of rows
-  $tableData = array();
-  while ($row = $result -> fetchArray (SQLITE3_NUM)) {
-    $row[0] = formatDate ($row[0]);
-    $tableData['data'][] = $row;
-  }
-
-  // Control no rows
-  if (empty($tableData['data'])) {
-    $tableData['data'] = '';
-  }
-
-  // Return json
-  echo (json_encode ($tableData));
-}
-
 ?>
diff --git a/front/pluginsCore.php b/front/pluginsCore.php
index 1c273f69..98d7da06 100755
--- a/front/pluginsCore.php
+++ b/front/pluginsCore.php
@@ -344,7 +344,7 @@ function createTabContent(pluginObj) {
   
   // Get data for events, objects, and history related to the plugin
   const objectData = getObjectData(prefix, colDefinitions, pluginObj);
-  const eventData = getEventData(prefix, colDefinitions);
+  const eventData = getEventData(prefix, colDefinitions, pluginObj);
   const historyData = getHistoryData(prefix, colDefinitions, pluginObj);
 
   // Append the content structure for the plugin's tab to the content location
@@ -378,10 +378,10 @@ function getColumnDefinitions(pluginObj) {
   return pluginObj["database_column_definitions"].filter(colDef => colDef.show);
 }
 
-function getEventData(prefix, colDefinitions) {
+function getEventData(prefix, colDefinitions, pluginObj) {
   // Extract event data specific to the plugin and format it for DataTables
   return pluginUnprocessedEvents
-    .filter(event => event.Plugin === prefix) // Filter events for the specific plugin
+    .filter(event => event.Plugin === prefix && shouldBeShown(event, pluginObj)) // Filter events for the specific plugin
     .map(event => colDefinitions.map(colDef => event[colDef.column] || '')); // Map to the defined columns
 }
 
@@ -395,7 +395,7 @@ function getObjectData(prefix, colDefinitions, pluginObj) {
 function getHistoryData(prefix, colDefinitions, pluginObj) {
   
   return pluginHistory
-  .filter(history => history.Plugin === prefix) // First, filter based on the plugin prefix
+  .filter(history => history.Plugin === prefix && shouldBeShown(history, pluginObj)) // First, filter based on the plugin prefix
     .sort((a, b) => b.Index - a.Index) // Then, sort by the Index field in descending order
     .slice(0, 50) // Limit the result to the first 50 entries
     .map(object => 
diff --git a/server/scan/device_handling.py b/server/scan/device_handling.py
index ee9d14fc..aa92d896 100755
--- a/server/scan/device_handling.py
+++ b/server/scan/device_handling.py
@@ -354,31 +354,45 @@ def create_new_devices (db):
     sql = db.sql # TO-DO
     startTime = timeNowTZ()
 
-    # Insert events for new devices from CurrentScan
-    mylog('debug','[New Devices] New devices - 1 Events')
+    # Insert events for new devices from CurrentScan (not yet in Devices)
 
-    query = f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
-                        eve_EventType, eve_AdditionalInfo,
-                        eve_PendingAlertEmail)
-                    SELECT cur_MAC, cur_IP, '{startTime}', 'New Device', cur_Vendor, 1
-                    FROM CurrentScan
-                    WHERE NOT EXISTS (SELECT 1 FROM Devices
-                                      WHERE devMac = cur_MAC)
-                """ 
 
+    mylog('debug', '[New Devices] Insert "New Device" Events')
+    query_new_device_events = f"""
+    INSERT INTO Events (
+        eve_MAC, eve_IP, eve_DateTime,
+        eve_EventType, eve_AdditionalInfo,
+        eve_PendingAlertEmail
+    )
+    SELECT cur_MAC, cur_IP, '{startTime}', 'New Device', cur_Vendor, 1
+    FROM CurrentScan
+    WHERE NOT EXISTS (
+        SELECT 1 FROM Devices
+        WHERE devMac = cur_MAC
+    )
+    """
     
-    mylog('debug',f'[New Devices] Log Events Query: {query}')
+    # mylog('debug',f'[New Devices] Log Events Query: {query_new_device_events}')
     
-    sql.execute(query)
+    sql.execute(query_new_device_events)
 
     mylog('debug',f'[New Devices] Insert Connection into session table')
 
-    sql.execute (f"""INSERT INTO Sessions (ses_MAC, ses_IP, ses_EventTypeConnection, ses_DateTimeConnection,
-                        ses_EventTypeDisconnection, ses_DateTimeDisconnection, ses_StillConnected, ses_AdditionalInfo)
-                    SELECT cur_MAC, cur_IP,'Connected','{startTime}', NULL , NULL ,1, cur_Vendor
-                    FROM CurrentScan 
-                    WHERE NOT EXISTS (SELECT 1 FROM Sessions
-                                      WHERE ses_MAC = cur_MAC) 
+    sql.execute (f"""INSERT INTO Sessions (
+                        ses_MAC, ses_IP, ses_EventTypeConnection, ses_DateTimeConnection,
+                        ses_EventTypeDisconnection, ses_DateTimeDisconnection,
+                        ses_StillConnected, ses_AdditionalInfo
+                    )
+                    SELECT cur_MAC, cur_IP, 'Connected', '{startTime}', NULL, NULL, 1, cur_Vendor
+                    FROM CurrentScan
+                    WHERE EXISTS (
+                        SELECT 1 FROM Devices
+                        WHERE devMac = cur_MAC
+                    )
+                    AND NOT EXISTS (
+                        SELECT 1 FROM Sessions
+                        WHERE ses_MAC = cur_MAC AND ses_StillConnected = 1
+                    )
                     """)
                     
     # Create new devices from CurrentScan
diff --git a/server/scan/session_events.py b/server/scan/session_events.py
index 2ab3c304..d4b1083b 100755
--- a/server/scan/session_events.py
+++ b/server/scan/session_events.py
@@ -48,10 +48,6 @@ def process_scan (db):
     mylog('verbose','[Process Scan] Updating Devices Info')
     update_devices_data_from_scan (db)
 
-    # Void false connection - disconnections
-    mylog('verbose','[Process Scan] Voiding false (ghost) disconnections')    
-    void_ghost_disconnections (db)
-
     # Pair session events (Connection / Disconnection)
     mylog('verbose','[Process Scan] Pairing session events (connection / disconnection) ')
     pair_sessions_events(db)  
@@ -75,104 +71,39 @@ def process_scan (db):
     # Commit changes    
     db.commitDB()
 
-#-------------------------------------------------------------------------------
-def void_ghost_disconnections (db):
-    sql = db.sql #TO-DO
-    startTime = timeNowTZ()
-    # Void connect ghost events (disconnect event exists in last X min.) 
-    mylog('debug','[Void Ghost Con] - 1 Connect ghost events')
-    sql.execute("""UPDATE Events SET eve_PairEventRowid = Null,
-                                eve_EventType ='VOIDED - ' || eve_EventType
-                            WHERE eve_MAC != 'Internet'
-                            AND eve_EventType in ('Connected', 'Down Reconnected')
-                            AND eve_DateTime = ?
-                            AND eve_MAC IN (
-                                SELECT Events.eve_MAC
-                                FROM CurrentScan, Devices, Events 
-                                WHERE devMac = cur_MAC
-                                    AND eve_MAC = cur_MAC
-                                    AND eve_EventType = 'Disconnected'
-                                    AND eve_DateTime >= DATETIME(?, '-3 minutes')
-                                ) """,
-                            (startTime, startTime))
-
-    # Void connect paired events
-    mylog('debug','[Void Ghost Con] - 2 Paired events')
-    sql.execute("""UPDATE Events SET eve_PairEventRowid = Null 
-                            WHERE eve_MAC != 'Internet'
-                            AND eve_PairEventRowid IN (
-                                SELECT Events.RowID
-                                FROM CurrentScan, Devices, Events 
-                                WHERE devMac = cur_MAC
-                                    AND eve_MAC = cur_MAC
-                                    AND eve_EventType = 'Disconnected'
-                                    AND eve_DateTime >= DATETIME(?, '-3 minutes')
-                                ) """,
-                            (startTime,))
-
-    # Void disconnect ghost events 
-    mylog('debug','[Void Ghost Con] - 3 Disconnect ghost events')
-    sql.execute("""UPDATE Events SET eve_PairEventRowid = Null, 
-                            eve_EventType = 'VOIDED - '|| eve_EventType
-                        WHERE eve_MAC != 'Internet'
-                        AND ROWID IN (
-                            SELECT Events.RowID
-                            FROM CurrentScan, Devices, Events 
-                            WHERE devMac = cur_MAC
-                                AND eve_MAC = cur_MAC
-                                AND eve_EventType = 'Disconnected'
-                                AND eve_DateTime >= DATETIME(?, '-3 minutes')
-                            ) """,
-                (startTime,))
-
-    mylog('debug','[Void Ghost Con] Void Ghost Connections end')
-
-    db.commitDB()
-
 #-------------------------------------------------------------------------------
 def pair_sessions_events (db):
-    # db.commitDB()
 
     sql = db.sql #TO-DO
-    mylog('debug', '[Pair Session] - START')
+    # Pair Connection / New Device events
 
-    # Step 1: Pair connection-related events with future unpaired disconnections
-    mylog('debug', '[Pair Session] - 1: Pair Connections → Disconnections')
-    sql.execute("""
-        UPDATE Events
-        SET eve_PairEventRowid = (
-            SELECT E2.ROWID
-            FROM Events AS E2
-            WHERE E2.eve_EventType IN ('Disconnected', 'Device Down')
-              AND E2.eve_MAC = Events.eve_MAC
-              AND E2.eve_PairEventRowid IS NULL
-              AND E2.eve_DateTime > Events.eve_DateTime
-            ORDER BY E2.eve_DateTime ASC
-            LIMIT 1
-        )
-        WHERE eve_EventType IN ('New Device', 'Connected', 'Down Reconnected')
-          AND eve_PairEventRowid IS NULL
-    """)
+    mylog('debug','[Pair Session] - 1 Connections / New Devices')
+    sql.execute ("""UPDATE Events
+                    SET eve_PairEventRowid =
+                       (SELECT ROWID
+                        FROM Events AS EVE2
+                        WHERE EVE2.eve_EventType IN ('New Device', 'Connected', 'Down Reconnected',
+                            'Device Down', 'Disconnected')
+                           AND EVE2.eve_MAC = Events.eve_MAC
+                           AND EVE2.eve_Datetime > Events.eve_DateTime
+                        ORDER BY EVE2.eve_DateTime ASC LIMIT 1)
+                    WHERE eve_EventType IN ('New Device', 'Connected', 'Down Reconnected')
+                    AND eve_PairEventRowid IS NULL
+                 """ )
 
-    # Step 2: Pair disconnection-related events with previous unpaired connections
-    mylog('debug', '[Pair Session] - 2: Pair Disconnections → Connections')
-    sql.execute("""
-        UPDATE Events
-        SET eve_PairEventRowid = (
-            SELECT E2.ROWID
-            FROM Events AS E2
-            WHERE E2.eve_EventType IN ('New Device', 'Connected', 'Down Reconnected')
-              AND E2.eve_MAC = Events.eve_MAC
-              AND E2.eve_PairEventRowid IS NULL
-              AND E2.eve_DateTime < Events.eve_DateTime
-            ORDER BY E2.eve_DateTime DESC
-            LIMIT 1
-        )
-        WHERE eve_EventType IN ('Disconnected', 'Device Down')
-          AND eve_PairEventRowid IS NULL
-    """)
+    # Pair Disconnection / Device Down
+    mylog('debug','[Pair Session] - 2 Disconnections')
+    sql.execute ("""UPDATE Events
+                    SET eve_PairEventRowid =
+                        (SELECT ROWID
+                         FROM Events AS EVE2
+                         WHERE EVE2.eve_PairEventRowid = Events.ROWID)
+                    WHERE eve_EventType IN ('Device Down', 'Disconnected')
+                      AND eve_PairEventRowid IS NULL
+                 """ )
 
-    mylog('debug', '[Pair Session] - END')
+
+    mylog('debug','[Pair Session] Pair session end')
     db.commitDB()