📊 Presence over time updates #816

docs/SUBNETS.md

@@ -2,7 +2,7 @@
 
 You need to specify the network interface and the network mask. You can also configure multiple subnets and specify VLANs (see VLAN exceptions below).
 
-`ARPSCAN` can scan multiple networks if the network allows it. To scan networks directly, the subnets must be accessible from the network where NetAlertX is running. You can verify this by running the following command in the container:
+`ARPSCAN` can scan multiple networks if the network allows it. To scan networks directly, the subnets must be accessible from the network where NetAlertX is running. This means NetAlertX needs to have access to the interface attached to that subnet. You can verify this by running the following command in the container:
 
 `sudo arp-scan --interface=eth0 192.168.1.0/24`
 

front/devices.php

@@ -15,8 +15,6 @@
 <?php
 
   require 'php/templates/header.php';
-  require 'php/templates/graph.php';
-  
 
   // check permissions
   $dbPath = "../db/app.db";
@@ -66,19 +64,37 @@
       </div>
       <script src="js/graph_online_history.js"></script>
       <script>
-        var pia_js_online_history_time = [<?php pia_graph_devices_data($Pia_Graph_Device_Time); ?>];
+        $.get('api/table_online_history.json?nocache=' + Date.now(), function(res) {
-        var pia_js_online_history_ondev = [<?php pia_graph_devices_data($Pia_Graph_Device_Online); ?>];
+              // Extracting data from the JSON response
-        var pia_js_online_history_dodev = [<?php pia_graph_devices_data($Pia_Graph_Device_Down); ?>];
+              var timeStamps = [];
-        var pia_js_online_history_ardev = [<?php pia_graph_devices_data($Pia_Graph_Device_Arch); ?>];
+              var onlineCounts = [];
+              var downCounts = [];
+              var offlineCounts = [];
+              var archivedCounts = [];
 
-        setTimeout(() => {
+              res.data.forEach(function(entry) {
-          pia_draw_graph_online_history(
+                  var dateObj = new Date(entry.Scan_Date);
-          pia_js_online_history_time, 
+                  var formattedTime = dateObj.toLocaleTimeString([], {hour: '2-digit', minute: '2-digit', hour12: false});
-          pia_js_online_history_ondev, 
-          pia_js_online_history_dodev, 
-          pia_js_online_history_ardev);
-        }, 500);
 
+                  timeStamps.push(formattedTime);
+                  onlineCounts.push(entry.Online_Devices);
+                  downCounts.push(entry.Down_Devices);
+                  offlineCounts.push(entry.Offline_Devices);
+                  archivedCounts.push(entry.Archived_Devices);
+              });
+
+              // Call your presenceOverTime function after data is ready
+              presenceOverTime(
+                  timeStamps,
+                  onlineCounts,
+                  offlineCounts,
+                  archivedCounts,
+                  downCounts
+              );
+          }).fail(function() {
+              // Handle any errors in fetching the data
+              console.error('Error fetching online history data.');
+          });
       </script>
 
 <!-- datatable ------------------------------------------------------------- -->

front/js/graph_online_history.js

@@ -1,13 +1,17 @@
-function pia_draw_graph_online_history(pia_js_graph_online_history_time, pia_js_graph_online_history_ondev, pia_js_graph_online_history_dodev, pia_js_graph_online_history_ardev) {
+function presenceOverTime(
-        var xValues = pia_js_graph_online_history_time;
+    timeStamp, 
-
+    onlineCount, 
-        // alert("dev presence")
+    offlineCount, 
+    archivedCount,
+    downCount
+  ) {
+        var xValues = timeStamp;
 
         // Data object for online status
         onlineData = {
           label: 'Online',
-          data: pia_js_graph_online_history_ondev,
+          data: onlineCount,
-          borderColor: "rgba(0, 166, 89)",
+          borderColor: "#00000",
           fill: true,
           backgroundColor: "rgba(0, 166, 89, .6)",
           pointStyle: 'circle',
@@ -15,20 +19,29 @@ function pia_draw_graph_online_history(pia_js_graph_online_history_time, pia_js_
           pointHoverRadius: 3
         };
 
+        // Data object for down status
+        downData = {          
+          label: 'Down',
+          data: downCount,
+          borderColor: "#00000",
+          fill: true,
+          backgroundColor: "#dd4b39",          
+        };
+
         // Data object for offline status
         offlineData = {          
-          label: 'Offline/Down',
+          label: 'Offline',
-          data: pia_js_graph_online_history_dodev,
+          data: offlineCount,
-          borderColor: "rgba(222, 74, 56)",
+          borderColor: "#00000",
           fill: true,
-          backgroundColor: "rgba(222, 74, 56, .6)",          
+          backgroundColor: "#b2b6be",          
         };
 
         // Data object for archived status
         archivedData = {
           label: 'Archived',
-          data: pia_js_graph_online_history_ardev,
+          data: archivedCount,
-          borderColor: "rgba(220,220,220)",
+          borderColor: "#00000",
           fill: true,
           backgroundColor: "rgba(220,220,220, .6)",
         };
@@ -42,23 +55,27 @@ function pia_draw_graph_online_history(pia_js_graph_online_history_time, pia_js_
         // Check if 'online' status should be displayed
         if(showStats.includes("online"))
         {
-          datasets.push(onlineData); // Add onlineData to datasets array
+          datasets.push(onlineData); 
+        }
+
+        // Check if 'down' status should be displayed
+        if(showStats.includes("down"))
+        {
+          datasets.push(downData); 
         }
 
         // Check if 'offline' status should be displayed
         if(showStats.includes("offline"))
         {
-          datasets.push(offlineData); // Add offlineData to datasets array
+          datasets.push(offlineData); 
         }
 
         // Check if 'archived' status should be displayed
         if(showStats.includes("archived"))
         {
-          datasets.push(archivedData); // Add archivedData to datasets array
+          datasets.push(archivedData); 
         }
 
-
-
         new Chart("OnlineChart", {
           type: "bar",
           scaleIntegersOnly: true,

front/php/templates/graph.php

@@ -1,59 +0,0 @@
-<?php
-
-  //------------------------------------------------------------------------------
-  // check if authenticated
-  require_once  $_SERVER['DOCUMENT_ROOT'] . '/php/templates/security.php';
-
-global $db;
-
-$Pia_Graph_Device_Time = array();
-$Pia_Graph_Device_All = array();
-$Pia_Graph_Device_Online = array();
-$Pia_Graph_Device_Down = array();
-$Pia_Graph_Device_Arch = array();
-
-$statusesToShow = "'online', 'offline', 'archived'";
-
-$statQuery = $db->query("SELECT * FROM Settings WHERE Code_Name = 'UI_PRESENCE'");
-
-while($r = $statQuery->fetchArray(SQLITE3_ASSOC))
-{
-  $statusesToShow =  $r['Value'];
-}
-
-$results = $db->query('SELECT * FROM Online_History ORDER BY Scan_Date DESC LIMIT 144');
-
-while ($row = $results->fetchArray()) 
-{
-    $time_raw = explode(' ', $row['Scan_Date']);
-    $time = explode(':', $time_raw[1]);
-    array_push($Pia_Graph_Device_Time, $time[0].':'.$time[1]);
-    
-    //  Offline
-    if(strpos($statusesToShow, 'offline') !== false) 
-    {
-      array_push($Pia_Graph_Device_Down, $row['Down_Devices']);
-    }
-
-    //  All
-    array_push($Pia_Graph_Device_All, $row['All_Devices']);
-
-    // Online
-    if(strpos($statusesToShow, 'online') !== false)
-    {
-      array_push($Pia_Graph_Device_Online, $row['Online_Devices']);
-    }
-    
-    // Archived
-    if(strpos($statusesToShow, 'archived') !== false)
-    {   
-      array_push($Pia_Graph_Device_Arch, $row['Archived_Devices']);
-    }
-}
-function pia_graph_devices_data($Pia_Graph_Array) {
-  $Pia_Graph_Array_rev = array_reverse($Pia_Graph_Array);
-  foreach ($Pia_Graph_Array_rev as $result) {
-      echo "'".$result."'";
-      echo ",";
-  }
-}

front/plugins/ui_settings/config.json

@@ -253,8 +253,8 @@
         ]
       },
       "maxLength": 50,
-      "default_value": ["online", "offline", "archived"],
+      "default_value": ["online", "down", "offline", "archived"],
-      "options": ["online", "offline", "archived"],
+      "options": ["online", "down", "offline", "archived"],
       "localized": [],
       "name": [
         {

front/presence.php

@@ -14,7 +14,6 @@
 
 <?php
   require 'php/templates/header.php';
-  require 'php/templates/graph.php';
 ?>
 
 <!-- Page ------------------------------------------------------------------ -->
@@ -128,19 +127,37 @@
 
       <script src="js/graph_online_history.js"></script>
       <script>
-        var pia_js_online_history_time = [<?php pia_graph_devices_data($Pia_Graph_Device_Time); ?>];
+        $.get('api/table_online_history.json?nocache=' + Date.now(), function(res) {
-        var pia_js_online_history_ondev = [<?php pia_graph_devices_data($Pia_Graph_Device_Online); ?>];
+              // Extracting data from the JSON response
-        var pia_js_online_history_dodev = [<?php pia_graph_devices_data($Pia_Graph_Device_Down); ?>];
+              var timeStamps = [];
-        var pia_js_online_history_ardev = [<?php pia_graph_devices_data($Pia_Graph_Device_Arch); ?>];
+              var onlineCounts = [];
+              var downCounts = [];
+              var offlineCounts = [];
+              var archivedCounts = [];
 
-        setTimeout(() => {
+              res.data.forEach(function(entry) {
-          pia_draw_graph_online_history(
+                  var dateObj = new Date(entry.Scan_Date);
-          pia_js_online_history_time, 
+                  var formattedTime = dateObj.toLocaleTimeString([], {hour: '2-digit', minute: '2-digit', hour12: false});
-          pia_js_online_history_ondev, 
-          pia_js_online_history_dodev, 
-          pia_js_online_history_ardev);
-        }, 500);
 
+                  timeStamps.push(formattedTime);
+                  onlineCounts.push(entry.Online_Devices);
+                  downCounts.push(entry.Down_Devices);
+                  offlineCounts.push(entry.Offline_Devices);
+                  archivedCounts.push(entry.Archived_Devices);
+              });
+
+              // Call your presenceOverTime function after data is ready
+              presenceOverTime(
+                  timeStamps,
+                  onlineCounts,
+                  offlineCounts,
+                  archivedCounts,
+                  downCounts
+              );
+          }).fail(function() {
+              // Handle any errors in fetching the data
+              console.error('Error fetching online history data.');
+          });
       </script>
   
       <!-- /.row -->

server/api.py

@@ -3,7 +3,7 @@ import json
 
 # Register NetAlertX modules 
 import conf  
-from const import (apiPath, sql_appevents, sql_devices_all, sql_events_pending_alert, sql_settings, sql_plugins_events, sql_plugins_history, sql_plugins_objects,sql_language_strings, sql_notifications_all)
+from const import (apiPath, sql_appevents, sql_devices_all, sql_events_pending_alert, sql_settings, sql_plugins_events, sql_plugins_history, sql_plugins_objects,sql_language_strings, sql_notifications_all, sql_online_history)
 from logger import mylog
 from helper import write_file
 
@@ -32,6 +32,7 @@ def update_api(db, all_plugins, isNotification = False, updateOnlyDataSources =
         ["plugins_objects", sql_plugins_objects],
         ["plugins_language_strings", sql_language_strings],
         ["notifications", sql_notifications_all],
+        ["online_history", sql_online_history],
         ["custom_endpoint", conf.API_CUSTOM_SQL],
     ]
 

server/const.py

@@ -38,6 +38,7 @@ sql_settings = "SELECT  * FROM Settings"
 sql_plugins_objects = "SELECT  * FROM Plugins_Objects"
 sql_language_strings = "SELECT  * FROM Plugins_Language_Strings"
 sql_notifications_all = "SELECT  * FROM Notifications"
+sql_online_history = "SELECT  * FROM Online_History"
 sql_plugins_events = "SELECT  * FROM Plugins_Events"
 sql_plugins_history = "SELECT  * FROM Plugins_History ORDER BY DateTimeChanged DESC"
 sql_new_devices = """SELECT * FROM ( 

server/database.py

@@ -115,6 +115,18 @@ class DB():
           );
           """)
 
+        # Offline_Devices column
+        Offline_Devices_missing = self.sql.execute ("""
+            SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Online_History') WHERE name='Offline_Devices'
+          """).fetchone()[0] == 0
+
+        if Offline_Devices_missing :
+          mylog('verbose', ["[upgradeDB] Adding Offline_Devices to the Online_History table"])
+          self.sql.execute("""
+            ALTER TABLE "Online_History" ADD "Offline_Devices" INTEGER
+          """)
+
+
         # -------------------------------------------------------------------------
         # Alter Devices table       
         # -------------------------------------------------------------------------
@@ -644,22 +656,3 @@ def get_all_devices(db):
 
 #-------------------------------------------------------------------------------
 
-#-------------------------------------------------------------------------------
-def insertOnlineHistory(db):
-    sql = db.sql #TO-DO
-    startTime = timeNowTZ()
-    # Add to History
-
-    History_All = db.read("SELECT * FROM Devices")
-    History_All_Devices  = len(History_All)
-
-    History_Archived = db.read("SELECT * FROM Devices WHERE dev_Archived = 1")
-    History_Archived_Devices  = len(History_Archived)
-
-    History_Online = db.read("SELECT * FROM Devices WHERE dev_PresentLastScan = 1")
-    History_Online_Devices  = len(History_Online)
-    History_Offline_Devices = History_All_Devices - History_Archived_Devices - History_Online_Devices
-
-    sql.execute ("INSERT INTO Online_History (Scan_Date, Online_Devices, Down_Devices, All_Devices, Archived_Devices) "+
-                 "VALUES ( ?, ?, ?, ?, ?)", (startTime, History_Online_Devices, History_Offline_Devices, History_All_Devices, History_Archived_Devices ) )
-    db.commitDB()

server/networkscan.py

@@ -2,7 +2,7 @@
 
 import conf
 
-from database import insertOnlineHistory
+
 from device import create_new_devices, print_scan_stats, save_scanned_devices, update_devices_data_from_scan
 from helper import timeNowTZ
 from logger import mylog
@@ -233,3 +233,44 @@ def insert_events (db):
                     WHERE dev_MAC = cur_MAC                        
                       AND dev_LastIP <> cur_IP """ )
     mylog('debug','[Events] - Events end')
+    
+    
+#-------------------------------------------------------------------------------
+def insertOnlineHistory(db):
+    sql = db.sql  # TO-DO: Implement sql object
+
+    scanTimestamp = timeNowTZ()
+
+    # Query to fetch all relevant device counts in one go
+    query = """
+    SELECT
+        COUNT(*) AS allDevics,
+        SUM(CASE WHEN dev_Archived = 1 THEN 1 ELSE 0 END) AS archivedDevices,
+        SUM(CASE WHEN dev_PresentLastScan = 1 THEN 1 ELSE 0 END) AS onlineDevices,
+        SUM(CASE WHEN dev_PresentLastScan = 0 AND dev_AlertDeviceDown = 1 THEN 1 ELSE 0 END) AS downDevices
+    FROM Devices
+    """
+    
+    deviceCounts = db.read(query)[0]  # Assuming db.read returns a list of rows, take the first (and only) row
+
+    allDevics = deviceCounts['allDevics']
+    archivedDevices = deviceCounts['archivedDevices']
+    onlineDevices = deviceCounts['onlineDevices']
+    downDevices = deviceCounts['downDevices']
+    
+    offlineDevices = allDevics - archivedDevices - onlineDevices
+
+    # Prepare the insert query using parameterized inputs
+    insert_query = """
+        INSERT INTO Online_History (Scan_Date, Online_Devices, Down_Devices, All_Devices, Archived_Devices, Offline_Devices)
+        VALUES (?, ?, ?, ?, ?, ?)
+    """
+    
+    mylog('debug', f'[Presence graph] Sql query: {insert_query} with values: {scanTimestamp}, {onlineDevices}, {downDevices}, {allDevics}, {archivedDevices}, {offlineDevices}')
+
+    # Insert the gathered data into the history table
+    sql.execute(insert_query, (scanTimestamp, onlineDevices, downDevices, allDevics, archivedDevices, offlineDevices))
+
+    db.commitDB()
+
+

server/plugin.py

@@ -129,7 +129,7 @@ def run_plugin_scripts(db, all_plugins, runType, pluginsState = plugins_state())
 
         if shouldRun:            
             # Header
-            updateState(f"Plugins: {prefix}")
+            updateState(f"Plugin: {prefix}")
                         
             print_plugin_info(plugin, ['display_name'])
             mylog('debug', ['[Plugins] CMD: ', get_plugin_setting_obj(plugin, "CMD")["value"]])