MQTT rework v0.1 + Settings UI improvements ⚙

front/css/pialert.css

@@ -623,6 +623,16 @@ height: 50px;
     display: none;
 }
 
+.settings-group
+{   
+    font-size: 20px;
+}
+
+.settings-group i{
+    font-size: 16px;
+}
+
+
 .table_row {    
     padding: 3px;    
     width:100%;

front/devices.php

@@ -397,41 +397,41 @@ function initializeDatatable (status) {
 
     // Convert JSON data into the desired format
     var dataArray = {
-      data: filteredData.map(function(item) {            
-        var originalRow = [
-          item.dev_Name || "",
-          item.dev_Owner || "",          
-          item.dev_DeviceType || "",
-          item.dev_Icon || "",
-          item.dev_Favorite || "",
-          item.dev_Group || "",
-          // ---
-          item.dev_FirstConnection || "",
-          item.dev_LastConnection || "",          
-          item.dev_LastIP || "",
-          item.dev_MAC || "",          // TODO handle internet node mac
-          getDeviceStatus(item) || "",
-          item.dev_MAC || "",      // hidden          
-          item.dev_LastIP || "",  // IP orderable
-          item.rowid || "",
-          item.dev_Network_Node_MAC_ADDR || "",
-          item.connected_devices || 0,
-          item.dev_Location || "",
-          item.dev_Vendor || "",
-          item.dev_Network_Node_port || 0
-        ]
+        data: filteredData.map(function(item) {
+            var originalRow = [
+                item.dev_Name || "",
+                item.dev_Owner || "",
+                item.dev_DeviceType || "",
+                item.dev_Icon || "",
+                item.dev_Favorite || "",
+                item.dev_Group || "",
+                // ---
+                item.dev_FirstConnection || "",
+                item.dev_LastConnection || "",
+                item.dev_LastIP || "",
+                (["2", "6", "A", "E", "a", "e"].includes(item.dev_MAC[1]) ? 1 : 0) || "", // Check if randomized MAC
+                getDeviceStatus(item) || "",
+                item.dev_MAC || "", // hidden
+                formatIPlong(item.dev_LastIP) || "", // IP orderable
+                item.rowid || "",
+                item.dev_Network_Node_MAC_ADDR || "",
+                item.connected_devices || 0,
+                item.dev_Location || "",
+                item.dev_Vendor || "",
+                item.dev_Network_Node_port || 0
+            ];
 
-        var newRow = []
-        
-        // reorder data based on user-definer columns order
-        for(index = 0; index < tableColumnOrder.length; index++)
-        {
-          newRow.push(originalRow[tableColumnOrder[index]]);
-        }
+            var newRow = [];
 
-        return newRow;      
-      })
+            // reorder data based on user-defined columns order
+            for (index = 0; index < tableColumnOrder.length; index++) {
+                newRow.push(originalRow[tableColumnOrder[index]]);
+            }
+
+            return newRow;
+        })
     };
+
     
     // TODO displayed columns
 
@@ -529,6 +529,7 @@ function initializeDatatable (status) {
         // Random MAC      
         {targets: [mapIndx(9)],
           'createdCell': function (td, cellData, rowData, row, col) {
+            console.log(cellData)
             if (cellData == 1){
               $(td).html ('<i data-toggle="tooltip" data-placement="right" title="Random MAC" style="font-size: 16px;" class="text-yellow glyphicon glyphicon-random"></i>');
             } else {

front/js/pialert_common.js

@@ -487,7 +487,21 @@ function getNameByMacAddress(macAddress) {
 }
 
 // -----------------------------------------------------------------------------
-// 
+// A function used to make the IP address orderable
+function formatIPlong(ipAddress) {
+  const parts = ipAddress.split('.');
+  if (parts.length !== 4) {
+      throw new Error('Invalid IP address format');
+  }
+  return (parseInt(parts[0]) << 24) |
+         (parseInt(parts[1]) << 16) |
+         (parseInt(parts[2]) << 8) |
+         parseInt(parts[3]);
+}
+
+// -----------------------------------------------------------------------------
+// A function to get a device property using the mac address as key and DB column nakme as parameter
+//  for the value to be returned
 function getDeviceDataByMacAddress(macAddress, dbColumn) {
 
   const sessionDataKey = 'devicesListAll_JSON';  

front/php/templates/language/en_us.json

@@ -72,7 +72,7 @@
         "Device_TableHead_FirstSession" : "First Session",
         "Device_TableHead_LastSession" : "Last Session",
         "Device_TableHead_LastIP" : "Last IP",
-        "Device_TableHead_MAC" : "MAC",
+        "Device_TableHead_MAC" : "Random MAC",
         "Device_TableHead_MAC_full" : "Full MAC",
         "Device_TableHead_LastIPOrder" : "Last IP Order",
         "Device_TableHead_Status" : "Status",
@@ -460,6 +460,14 @@
         "Plugins_Filters_Mac" : "Mac Filter",
         "Plugins_Out_of" : "out of",        
         "Plugins_no_control" : "No form control was found to render this value.",        
+        "Plugins_type_core" : "Core",        
+        "Plugins_type_core_icon" : "fa-solid fa-gem",        
+        "Plugins_type_scanner" : "Scanners",        
+        "Plugins_type_scanner_icon" : "fa-solid fa-magnifying-glass-plus",        
+        "Plugins_type_system" : "System",        
+        "Plugins_type_system_icon" : "fa-solid fa-gear",        
+        "Plugins_type_publisher" : "Publishers",        
+        "Plugins_type_publisher_icon" : "fa-solid fa-comment-dots",        
         "Settings_Metadata_Toggle" : "Show/hide metadata for the given setting.",
         "settings_missing" : "Not all settings loaded, refresh the page! This is probably caused by a high load on the database or app startup sequence.",
         "settings_missing_block" : "You can not save your settings without specifying all setting keys. Refresh the page. This is probably caused by a high load on the database.",
@@ -546,21 +554,8 @@
         "MQTT_display_name" : "MQTT",
         "MQTT_icon" : "<i class=\"fa fa-square-rss\"></i>",
         "REPORT_TITLE" : "Report",
-        "REPORT_ERROR" : "The page you are looking for is temporarily unavailable, please try again after a few seconds",
-        "REPORT_MQTT_name" : "Enable MQTT",
-        "REPORT_MQTT_description" : "Enable sending notifications via <a target=\"_blank\" href=\"https://www.home-assistant.io/integrations/mqtt/\">MQTT</a> to your Home Assistance instance.",
-        "MQTT_BROKER_name" : "MQTT broker URL",
-        "MQTT_BROKER_description" : "MQTT host URL (do not include <code>http://</code> or <code>https://</code>).",
-        "MQTT_PORT_name" : "MQTT broker port",
-        "MQTT_PORT_description" : "Port number where the broker is listening. Usually <code>1883</code>.",
-        "MQTT_USER_name" : "MQTT user",
-        "MQTT_USER_description" : "User name used to login into your MQTT broker instance.",
-        "MQTT_PASSWORD_name" : "MQTT password",
-        "MQTT_PASSWORD_description" : "Password used to login into your MQTT broker instance.",
-        "MQTT_QOS_name" : "MQTT Quality of Service",
-        "MQTT_QOS_description" : "Quality of service setting for MQTT message sending. <code>0</code> - Low quality to <code>2</code> - High quality. The higher the quality the longer the delay.",
-        "MQTT_DELAY_SEC_name" : "MQTT delay per device",
-        "MQTT_DELAY_SEC_description" : "A little hack - delay adding to the queue in case the process is restarted and previous publish processes aborted (it takes ~<code>2</code>s to update a sensor config on the broker). Tested with <code>2</code>-<code>3</code> seconds of delay. This delay is only applied when devices are created (during the first notification loop). It doesn not affect subsequent scans or notifications.",
+        "REPORT_ERROR" : "The page you are looking for is temporarily unavailable, please try again after a few seconds",       
+
         "API_display_name" : "API",
         "API_icon" : "<i class=\"fa fa-arrow-down-up-across-line\"></i>",
         "API_CUSTOM_SQL_name" : "Custom endpoint",

front/plugins/_publisher_apprise/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "_publisher_apprise",
     "unique_prefix": "APPRISE",
+    "plugin_type": "publisher",
     "enabled": true,    
     "data_source":  "script",
     "show_ui": true,

front/plugins/_publisher_email/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "_publisher_email",
     "unique_prefix": "SMTP",
+    "plugin_type": "publisher",
     "enabled": true,
     "data_source": "script",
     "show_ui": true,

front/plugins/_publisher_mqtt/README.md

@@ -0,0 +1,8 @@
+## Overview
+
+- TBC
+
+### Usage
+
+- Go to settings and fill in relevant details.
+

front/plugins/_publisher_mqtt/config.json

@@ -0,0 +1,471 @@
+{
+    "code_name": "_publisher_mqtt",
+    "unique_prefix": "MQTT",
+    "plugin_type": "publisher",
+    "enabled": true,    
+    "data_source":  "script",
+    "show_ui": true,
+    "localized": ["display_name", "description", "icon"],
+	"display_name" : [
+        {
+		  "language_code": "en_us",
+		  "string" : "MQTT publisher"
+		},
+        {
+		  "language_code": "es_es",
+		  "string" : "Habilitar MQTT"
+		}
+    ],
+    "icon":[{
+        "language_code": "en_us",
+        "string" : "<i class=\"fa-solid fa-square-rss\"></i>"
+		}],     
+    "description": [
+        {
+        "language_code": "en_us",
+        "string" : "A plugin to publish a notification via the Apprise gateway."
+		}
+    ],
+    "params" : [],  
+    "database_column_definitions":
+    [          
+        {
+            "column": "Index",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "Plugin",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "Object_PrimaryID",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "url",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				}]
+        },
+        {
+            "column": "Object_SecondaryID",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "DateTimeCreated",
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Sent when"
+				}]
+        },
+        {
+            "column": "DateTimeChanged",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Changed"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Cambiado"					
+                }]
+        },
+        {
+            "column": "Watched_Value1",
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Notification GUID"
+				}]
+        },
+        {
+            "column": "Watched_Value2",
+            "css_classes": "col-sm-8",
+            "show": true,
+            "type": "textarea_readonly",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Result"
+				}]
+        },
+        {
+            "column": "Watched_Value3",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "Watched_Value4",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },       
+        {
+            "column": "UserData",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "textbox_save",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Comments"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Comentarios"					
+                }]
+        },        
+        {
+            "column": "Status",
+            "css_classes": "col-sm-1",
+            "show": false,
+            "type": "replace",            
+            "default_value":"",
+            "options": [
+                {
+                    "equals": "watched-not-changed",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-square-check'></i><div></div>"
+                },
+                {
+                    "equals": "watched-changed",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-triangle-exclamation'></i></div>"
+                },
+                {
+                    "equals": "new",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
+                },
+                {
+                    "equals": "missing-in-last-scan",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
+                }
+            ],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Status"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Estado"					
+                }]
+        },        
+        {
+            "column": "Extra",
+            "css_classes": "col-sm-3",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Extra"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Extra"					
+                }]
+        }            
+    ],
+    "settings":[
+        {
+            "function": "RUN",         
+            "events": ["test"],   
+            "type": "text.select",            
+            "default_value":"disabled",
+            "options": ["disabled", "on_notification" ],
+            "localized": ["name", "description"],
+            "name" :[{
+                "language_code": "en_us",
+                "string" : "When to run"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Cuando ejecuta"
+            }],
+            "description": [
+                {
+                "language_code": "en_us",
+                "string" : "Enable sending notifications via <a target=\"_blank\" href=\"https://www.home-assistant.io/integrations/mqtt/\">MQTT</a> to your Home Assistance instance."
+				},
+                {
+                "language_code": "es_es",
+                "string" : "Habilitar el envío de notificaciones a través de <a target=\"_blank\" href=\"https://www.home-assistant.io/integrations/mqtt/\">MQTT</a> a su Home Assistance."
+				}
+            ]
+        },
+        {
+            "function": "CMD",
+            "type": "readonly",
+            "default_value":"python3 /home/pi/pialert/front/plugins/_publisher_mqtt/mqtt.py",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "Command"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Comando"						
+            }],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Command to run"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Comando a ejecutar"	
+            }]
+        },
+        {
+            "function": "RUN_TIMEOUT",
+            "type": "integer",
+            "default_value": 600,
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "Run timeout"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Tiempo de espera de ejecución"				
+				},
+				{
+                "language_code": "de_de",
+                "string" : "Wartezeit"
+            }],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Tiempo máximo en segundos para esperar a que finalice el script. Si se supera este tiempo, el script se cancela."
+            }]
+        },
+        {
+            "function": "BROKER",
+            "type": "text",
+            "default_value": "",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT broker URL"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "URL del broker MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "MQTT host URL (do not include <code>http://</code> or <code>https://</code>)."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "URL del host MQTT (no incluya <code>http://</code> o <code>https://</code>)."
+            }]
+        },
+        {
+            "function": "PORT",
+            "type": "integer",
+            "default_value": 1883,
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT broker port"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Puerto del broker MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Port number where the broker is listening. Usually <code>1883</code>."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Puerto donde escucha el broker MQTT. Normalmente <code>1883</code>."
+                }]
+        },
+        {
+            "function": "USER",
+            "type": "text",
+            "default_value": "",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT user"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Usuario de MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "User name used to login into your MQTT broker instance."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Nombre de usuario utilizado para iniciar sesión en su instancia de agente de MQTT."
+            }]
+        },
+        {
+            "function": "PASSWORD",
+            "type": "password",
+            "default_value": "",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT password"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Contraseña de MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Password used to login into your MQTT broker instance."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Contraseña utilizada para iniciar sesión en su instancia de agente de MQTT."
+            }]
+        },
+        {
+            "function": "QOS",
+            "type": "integer.select",
+            "default_value": 0,
+            "options": [0, 1, 2],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT Quality of Service"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Calidad de servicio MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Quality of service setting for MQTT message sending. <code>0</code> - Low quality to <code>2</code> - High quality. The higher the quality the longer the delay."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Configuración de calidad de servicio para el envío de mensajes MQTT. <code>0</code>: baja calidad a <code>2</code>: alta calidad. Cuanto mayor sea la calidad, mayor será el retraso."
+            }]
+        },
+        {
+            "function": "DELAY_SEC",
+            "type": "integer",
+            "default_value": 2,
+            "options": [2, 3, 4, 5],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT delay per device"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Retraso de MQTT por dispositivo"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "A little hack - delay adding to the queue in case the process is restarted and previous publish processes aborted (it takes ~<code>2</code>s to update a sensor config on the broker). Tested with <code>2</code>-<code>3</code> seconds of delay. This delay is only applied when devices are created (during the first notification loop). It doesn not affect subsequent scans or notifications."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Un pequeño truco: retrase la adición a la cola en caso de que el proceso se reinicie y los procesos de publicación anteriores se anulen (se necesitan ~<code>2</code>s para actualizar la configuración de un sensor en el intermediario). Probado con <code>2</code>-<code>3</code> segundos de retraso. Este retraso solo se aplica cuando se crean dispositivos (durante el primer bucle de notificación). No afecta los escaneos o notificaciones posteriores."
+            }]
+        }         
+    ]
+}

front/plugins/_publisher_mqtt/ignore_plugin

@@ -1 +0,0 @@
-This plugin will not be loaded

front/plugins/_publisher_mqtt/mqtt.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+
+import json
+import subprocess
+import argparse
+import os
+import pathlib
+import sys
+from datetime import datetime
+import time
+import re
+from paho.mqtt import client as mqtt_client
+
+
+# Replace these paths with the actual paths to your Pi.Alert directories
+sys.path.extend(["/home/pi/pialert/front/plugins", "/home/pi/pialert/pialert"])
+
+#  PiAlert modules
+import conf
+from plugin_helper import Plugin_Objects
+from logger import mylog, append_line_to_file
+from helper import timeNowTZ, noti_obj, get_setting_value, bytes_to_string, sanitize_string
+from notification import Notification_obj
+from database import DB, get_all_devices, get_device_stats
+
+
+CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
+RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
+
+pluginName = 'MQTT'
+
+def main():
+    
+    mylog('verbose', [f'[{pluginName}](publisher) In script'])    
+    
+    # Check if basic config settings supplied
+    if check_config() == False:
+        mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
+        return
+
+    # Create a database connection
+    db = DB()  # instance of class DB
+    db.open()
+
+    mqtt_start(db)
+
+
+
+#-------------------------------------------------------------------------------
+# MQTT
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+def check_config():
+        if get_setting_value('MQTT_BROKER') == '' or get_setting_value('MQTT_PORT') == '' or get_setting_value('MQTT_USER') == '' or get_setting_value('MQTT_PASSWORD') == '':
+            mylog('none', ['[Check Config] Error: MQTT service not set up correctly. Check your pialert.conf MQTT_* variables.'])
+            return False
+        else:
+            return True
+
+
+#-------------------------------------------------------------------------------
+class sensor_config:
+    def __init__(self, deviceId, deviceName, sensorType, sensorName, icon):
+        self.deviceId = deviceId
+        self.deviceName = deviceName
+        self.sensorType = sensorType
+        self.sensorName = sensorName
+        self.icon = icon 
+        self.hash = str(hash(str(deviceId) + str(deviceName)+ str(sensorType)+ str(sensorName)+ str(icon)))
+
+#-------------------------------------------------------------------------------
+
+def publish_mqtt(client, topic, message):
+    status = 1
+    while status != 0:
+        result = client.publish(
+                topic=topic,
+                payload=message,
+                qos=get_setting_value('MQTT_QOS'),
+                retain=True,
+            )
+
+        status = result[0]
+
+        if status != 0:            
+            mylog('minimal', [f"[{pluginName}] Waiting to reconnect to MQTT broker"])
+            time.sleep(0.1) 
+    return True
+
+#-------------------------------------------------------------------------------
+def create_generic_device(client):  
+    
+    deviceName = 'PiAlert'
+    deviceId = 'pialert'    
+    
+    create_sensor(client, deviceId, deviceName, 'sensor', 'online', 'wifi-check')    
+    create_sensor(client, deviceId, deviceName, 'sensor', 'down', 'wifi-cancel')        
+    create_sensor(client, deviceId, deviceName, 'sensor', 'all', 'wifi')
+    create_sensor(client, deviceId, deviceName, 'sensor', 'archived', 'wifi-lock')
+    create_sensor(client, deviceId, deviceName, 'sensor', 'new', 'wifi-plus')
+    create_sensor(client, deviceId, deviceName, 'sensor', 'unknown', 'wifi-alert')
+        
+
+#-------------------------------------------------------------------------------
+def create_sensor(client, deviceId, deviceName, sensorType, sensorName, icon):    
+
+    new_sensor_config = sensor_config(deviceId, deviceName, sensorType, sensorName, icon)
+
+    # check if config already in list and if not, add it, otherwise skip
+    is_unique = True
+
+    mylog('minimal', [f"[{pluginName}] Already previously published sensors: {len(conf.mqtt_sensors)}"])
+
+    for sensor in conf.mqtt_sensors:
+        if sensor.hash == new_sensor_config.hash:
+            is_unique = False
+            break
+           
+    # save if unique
+    if is_unique:             
+        publish_sensor(client, new_sensor_config)        
+
+
+
+
+#-------------------------------------------------------------------------------
+def publish_sensor(client, sensorConf):         
+
+    message = '{ \
+                "name":"'+ sensorConf.deviceName +' '+sensorConf.sensorName+'", \
+                "state_topic":"system-sensors/'+sensorConf.sensorType+'/'+sensorConf.deviceId+'/state", \
+                "value_template":"{{value_json.'+sensorConf.sensorName+'}}", \
+                "unique_id":"'+sensorConf.deviceId+'_sensor_'+sensorConf.sensorName+'", \
+                "device": \
+                    { \
+                        "identifiers": ["'+sensorConf.deviceId+'_sensor"], \
+                        "manufacturer": "PiAlert", \
+                        "name":"'+sensorConf.deviceName+'" \
+                    }, \
+                "icon":"mdi:'+sensorConf.icon+'" \
+                }'
+
+    topic='homeassistant/'+sensorConf.sensorType+'/'+sensorConf.deviceId+'/'+sensorConf.sensorName+'/config'
+
+    # add the sensor to the global list to keep track of succesfully added sensors
+    if publish_mqtt(client, topic, message):        
+                                     # hack - delay adding to the queue in case the process is 
+        time.sleep(get_setting_value('MQTT_DELAY_SEC'))   # restarted and previous publish processes aborted 
+                                     # (it takes ~2s to update a sensor config on the broker)
+        conf.mqtt_sensors.append(sensorConf)
+
+#-------------------------------------------------------------------------------
+def mqtt_create_client():
+    def on_disconnect(client, userdata, rc):
+        conf.mqtt_connected_to_broker = False
+        
+        # not sure is below line is correct / necessary        
+        # client = mqtt_create_client() 
+
+    def on_connect(client, userdata, flags, rc):
+        
+        if rc == 0: 
+            mylog('verbose', [f"[{pluginName}]         Connected to broker"])            
+            conf.mqtt_connected_to_broker = True     # Signal connection 
+        else: 
+            mylog('none', [f"[{pluginName}]         Connection failed"])
+            conf.mqtt_connected_to_broker = False
+
+
+    client = mqtt_client.Client('PiAlert')   # Set Connecting Client ID    
+    client.username_pw_set(get_setting_value('MQTT_USER'), get_setting_value('MQTT_PASSWORD'))    
+    client.on_connect = on_connect
+    client.on_disconnect = on_disconnect
+    client.connect(get_setting_value('MQTT_BROKER'), get_setting_value('MQTT_PORT'))
+    client.loop_start() 
+
+    return client
+
+#-------------------------------------------------------------------------------
+def mqtt_start(db):    
+
+    #global client
+
+    if conf.mqtt_connected_to_broker == False:
+        conf.mqtt_connected_to_broker = True           
+        conf.client = mqtt_create_client() 
+    
+    client = conf.client
+    # General stats    
+
+    # Create a generic device for overal stats
+    create_generic_device(client)
+
+    # Get the data
+    row = get_device_stats(db)   
+
+    columns = ["online","down","all","archived","new","unknown"]
+
+    payload = ""
+
+    # Update the values 
+    for column in columns:       
+        payload += '"'+column+'": ' + str(row[column]) +','       
+
+    # Publish (warap into {} and remove last ',' from above)
+    publish_mqtt(client, "system-sensors/sensor/pialert/state",              
+            '{ \
+                '+ payload[:-1] +'\
+            }'
+        )
+
+
+    # Specific devices
+
+    # Get all devices
+    devices = get_all_devices(db)
+
+    sec_delay = len(devices) * int(get_setting_value('MQTT_DELAY_SEC'))*5
+
+    mylog('minimal', [f"[{pluginName}]         Estimated delay: ", (sec_delay), 's ', '(', round(sec_delay/60,1) , 'min)' ])
+
+    for device in devices:        
+
+        # Create devices in Home Assistant - send config messages
+        deviceId = 'mac_' + device["dev_MAC"].replace(" ", "").replace(":", "_").lower()
+        deviceNameDisplay = re.sub('[^a-zA-Z0-9-_\s]', '', device["dev_Name"]) 
+
+        create_sensor(client, deviceId, deviceNameDisplay, 'sensor', 'last_ip', 'ip-network')
+        create_sensor(client, deviceId, deviceNameDisplay, 'binary_sensor', 'is_present', 'wifi')
+        create_sensor(client, deviceId, deviceNameDisplay, 'sensor', 'mac_address', 'folder-key-network')
+        create_sensor(client, deviceId, deviceNameDisplay, 'sensor', 'is_new', 'bell-alert-outline')
+        create_sensor(client, deviceId, deviceNameDisplay, 'sensor', 'vendor', 'cog')
+    
+        # update device sensors in home assistant              
+
+        publish_mqtt(client, 'system-sensors/sensor/'+deviceId+'/state', 
+            '{ \
+                "last_ip": "' + device["dev_LastIP"] +'", \
+                "is_new": "' + str(device["dev_NewDevice"]) +'", \
+                "vendor": "' + sanitize_string(device["dev_Vendor"]) +'", \
+                "mac_address": "' + str(device["dev_MAC"]) +'" \
+            }'
+        ) 
+
+        publish_mqtt(client, 'system-sensors/binary_sensor/'+deviceId+'/state', 
+            '{ \
+                "is_present": "' + to_binary_sensor(str(device["dev_PresentLastScan"])) +'"\
+            }'
+        ) 
+
+        # delete device / topic
+        #  homeassistant/sensor/mac_44_ef_bf_c4_b1_af/is_present/config
+        # client.publish(
+        #     topic="homeassistant/sensor/"+deviceId+"/is_present/config",
+        #     payload="",
+        #     qos=1,
+        #     retain=True,
+        # )        
+    # time.sleep(10)
+
+
+#===============================================================================
+# Home Assistant UTILs
+#===============================================================================
+def to_binary_sensor(input):
+    # In HA a binary sensor returns ON or OFF    
+    result = "OFF"
+
+    # bytestring
+    if isinstance(input, str):
+        if input == "1":
+            result = "ON"
+    elif isinstance(input, int):
+        if input == 1:
+            result = "ON"
+    elif isinstance(input, bool):
+        if input == True:
+            result = "ON"
+    elif isinstance(input, bytes):
+        if bytes_to_string(input) == "1":
+            result = "ON"
+    return result
+
+
+
+
+#  -------------INIT---------------------
+if __name__ == '__main__':
+    sys.exit(main())
+
+
+

front/plugins/arp_scan/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "arp_scan",
   "unique_prefix": "ARPSCAN",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "CurrentScan",

front/plugins/csv_backup/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "csv_backup",
   "unique_prefix": "CSVBCKP",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",  
   "show_ui": false,

front/plugins/db_cleanup/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "db_cleanup",
   "unique_prefix": "DBCLNP",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",  
   "show_ui": false,

front/plugins/ddns_update/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "ddns_update",
   "unique_prefix": "DDNS",
+  "plugin_type": "system",
   "enabled": true,
   "data_filters": [
     {

front/plugins/dhcp_leases/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "dhcp_leases",
   "unique_prefix": "DHCPLSS",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "data_filters": [

front/plugins/dhcp_servers/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "dhcp_servers",
     "unique_prefix": "DHCPSRVS",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "show_ui": true,

front/plugins/internet_ip/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "internet_ip",
   "unique_prefix": "INTRNT",
+  "plugin_type": "scanner",
   "enabled": true,
   "mapped_to_table": "CurrentScan",
   "data_filters": [

front/plugins/internet_speedtest/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "internet_speedtest",
   "unique_prefix": "INTRSPD",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "show_ui": true,

front/plugins/known_template/config.json

@@ -2,6 +2,7 @@
   "code_name": "known_template",
   "template_type": "database-entry",
   "unique_prefix": "KNWN",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "template", 
   "show_ui": false, 

front/plugins/newdev_template/config.json

@@ -2,6 +2,7 @@
   "code_name": "Devices.new",
   "template_type": "database-entry",
   "unique_prefix": "NEWDEV",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "template",  
   "show_ui": false,

front/plugins/nmap_scan/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "nmap_scan",
     "unique_prefix": "NMAP",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "data_filters": [

front/plugins/pholus_scan/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "pholus_scan",
   "unique_prefix": "PHOLUS",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "Pholus_Scan",

front/plugins/pihole_scan/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "pihole_scan",
   "unique_prefix": "PIHOLE",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "sqlite-db-query",
   "mapped_to_table": "CurrentScan",  

front/plugins/set_password/config.json

@@ -2,6 +2,7 @@
   "code_name": "set_password",
   "template_type": "database-entry",
   "unique_prefix": "SETPWD",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",  
   "show_ui": false,

front/plugins/snmp_discovery/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "snmp_discovery",
     "unique_prefix": "SNMPDSC",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "data_filters": [

front/plugins/undiscoverables/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "undiscoverables",
   "unique_prefix": "UNDIS",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "CurrentScan",

front/plugins/unifi_import/config.json

@@ -1,5 +1,8 @@
 {
     "code_name": "unifi_import",
+    "show_ui": true,
+    "unique_prefix": "UNFIMP",
+    "plugin_type": "scanner",
     "data_filters": [
         {
             "compare_column": "Object_PrimaryID",
@@ -950,7 +953,5 @@
         ],
         "type": "text.select"
         }
-    ],
-    "show_ui": true,
-    "unique_prefix": "UNFIMP"
+    ]    
 }

front/plugins/vendor_update/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "vendor_update",
   "unique_prefix": "VNDRPDT",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",
   "show_ui": true,

front/plugins/website_monitor/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "website_monitor",
     "unique_prefix": "WEBMON",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "show_ui": true,

front/settings.php

@@ -72,8 +72,30 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
       </h1>
       <div class="settingsImported"><?= lang("settings_imported");?> <span id="lastImportedTime"></span></div>      
     </section>
-    <div class="content settingswrap" id='accordion_gen'>
-     <!-- PLACEHOLDER -->
+    <div class="content settingswrap col-sm-12" id="accordion_gen">
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_core");?> (<i class="<?= lang("Plugins_type_core_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="core_content"></div>
+
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_system");?> (<i class="<?= lang("Plugins_type_system_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="system_content"></div>
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_scanner");?> (<i class="<?= lang("Plugins_type_scanner_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="scanner_content"></div>
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_publisher");?> (<i class="<?= lang("Plugins_type_publisher_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="publisher_content"></div>
+
+     
    </div>
    
     <!-- /.content -->
@@ -96,6 +118,25 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
 
 <script>
 
+  // Get plugin type base on prefix
+  function getPluginType(pluginsData, prefix)
+  {
+
+    result = "core_content"
+
+    pluginsData.forEach((plug) => {
+
+      if (plug.unique_prefix == prefix ) {
+        id = plug.plugin_type + "_content"
+        
+        console.log(id)
+        result = plug.plugin_type + "_content";        
+      }
+    });
+
+    return result;
+  }
+
   
   function getData(){
 
@@ -103,18 +144,23 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
         
         settingsData = res["data"];       
 
-        initSettingsPage(settingsData);
+        $.get('api/plugins.json?nocache=' + Date.now(), function(res) {  
+
+          pluginsData = res["data"];  
+
+          initSettingsPage(settingsData, pluginsData);
+        })
     })
   }
 
-  function initSettingsPage(settingsData){
+  function initSettingsPage(settingsData, pluginsData){
 
     const settingGroups = [];
+    const pluginTypes = [];
     const settingKeyOfLists = [];
     // core groups are the ones not generated by plugins
     const settingCoreGroups = ['General'];
 
-
     // Loop through the settingsArray and collect unique settingGroups
     settingsData.forEach((set) => {
       if (!settingGroups.includes(set.Group)) {
@@ -125,7 +171,7 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
 
     console.log(settingGroups);
 
-    let headersHtml = '';    
+    
     let pluginHtml = `
           <div class="row table_row">
             <div class="table_cell bold">
@@ -140,19 +186,11 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
 
     for (const group of settingGroups) {
       let isPlugin = false;
-      let settingGroupTypeHtml = '';
 
-      if (settingCoreGroups.includes(group)) {
-        settingGroupTypeHtml = '';
-      } else {
-        settingGroupTypeHtml = ' (<i class="fa-regular fa-plug fa-sm"></i>) ';
-        isPlugin = true;
-      }
-
-      headersHtml += `<div class="box panel panel-default">
+      headerHtml = `<div class="box panel panel-default">
                   <a data-toggle="collapse" data-parent="#accordion_gen" href="#${group}">
                     <div class="panel-heading">
-                      <h4 class="panel-title">${getString(group+"_icon")} ${getString(group+"_display_name")} ${settingGroupTypeHtml}</h4>
+                      <h4 class="panel-title">${getString(group+"_icon")} ${getString(group+"_display_name")}</h4>
                     </div>
                   </a>
                   <div id="${group}" data-myid="collapsible" class="panel-collapse collapse ${isIn}">
@@ -163,10 +201,11 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
                 </div>
                     `;
       isIn = ' '; // open the first panel only by default on page load
-    }
-  
-    // generate headers/sections
-    $('#accordion_gen').html(headersHtml);
+
+      // generate headers/sections      
+      $('#'+getPluginType(pluginsData, group)).append(headerHtml);
+    }  
+
 
     // generate panel content
     for (const group of settingGroups) {
@@ -353,7 +392,7 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
           const eventsList = createArray(set['Events']);          
 
           if (eventsList.length > 0) {
-            console.log(eventsList)
+            // console.log(eventsList)
             eventsList.forEach(event => {
               eventsHtml += `<span class="input-group-addon pointer"
                 data-myparam="${codeName}"