MQTT rework v0.1 + Settings UI improvements ⚙

front/css/pialert.css

@@ -623,6 +623,16 @@ height: 50px;
     display: none;
 }
 
+.settings-group
+{   
+    font-size: 20px;
+}
+
+.settings-group i{
+    font-size: 16px;
+}
+
+
 .table_row {    
     padding: 3px;    
     width:100%;

front/devices.php

@@ -397,42 +397,42 @@ function initializeDatatable (status) {
 
     // Convert JSON data into the desired format
     var dataArray = {
-      data: filteredData.map(function(item) {            
+        data: filteredData.map(function(item) {
-        var originalRow = [
+            var originalRow = [
-          item.dev_Name || "",
+                item.dev_Name || "",
-          item.dev_Owner || "",          
+                item.dev_Owner || "",
-          item.dev_DeviceType || "",
+                item.dev_DeviceType || "",
-          item.dev_Icon || "",
+                item.dev_Icon || "",
-          item.dev_Favorite || "",
+                item.dev_Favorite || "",
-          item.dev_Group || "",
+                item.dev_Group || "",
-          // ---
+                // ---
-          item.dev_FirstConnection || "",
+                item.dev_FirstConnection || "",
-          item.dev_LastConnection || "",          
+                item.dev_LastConnection || "",
-          item.dev_LastIP || "",
+                item.dev_LastIP || "",
-          item.dev_MAC || "",          // TODO handle internet node mac
+                (["2", "6", "A", "E", "a", "e"].includes(item.dev_MAC[1]) ? 1 : 0) || "", // Check if randomized MAC
-          getDeviceStatus(item) || "",
+                getDeviceStatus(item) || "",
-          item.dev_MAC || "",      // hidden          
+                item.dev_MAC || "", // hidden
-          item.dev_LastIP || "",  // IP orderable
+                formatIPlong(item.dev_LastIP) || "", // IP orderable
-          item.rowid || "",
+                item.rowid || "",
-          item.dev_Network_Node_MAC_ADDR || "",
+                item.dev_Network_Node_MAC_ADDR || "",
-          item.connected_devices || 0,
+                item.connected_devices || 0,
-          item.dev_Location || "",
+                item.dev_Location || "",
-          item.dev_Vendor || "",
+                item.dev_Vendor || "",
-          item.dev_Network_Node_port || 0
+                item.dev_Network_Node_port || 0
-        ]
+            ];
 
-        var newRow = []
+            var newRow = [];
 
-        // reorder data based on user-definer columns order
+            // reorder data based on user-defined columns order
-        for(index = 0; index < tableColumnOrder.length; index++)
+            for (index = 0; index < tableColumnOrder.length; index++) {
-        {
+                newRow.push(originalRow[tableColumnOrder[index]]);
-          newRow.push(originalRow[tableColumnOrder[index]]);
+            }
-        }
 
-        return newRow;      
+            return newRow;
-      })
+        })
     };
 
+    
     // TODO displayed columns
 
 
@@ -529,6 +529,7 @@ function initializeDatatable (status) {
         // Random MAC      
         {targets: [mapIndx(9)],
           'createdCell': function (td, cellData, rowData, row, col) {
+            console.log(cellData)
             if (cellData == 1){
               $(td).html ('<i data-toggle="tooltip" data-placement="right" title="Random MAC" style="font-size: 16px;" class="text-yellow glyphicon glyphicon-random"></i>');
             } else {

front/js/pialert_common.js

@@ -487,7 +487,21 @@ function getNameByMacAddress(macAddress) {
 }
 
 // -----------------------------------------------------------------------------
-// 
+// A function used to make the IP address orderable
+function formatIPlong(ipAddress) {
+  const parts = ipAddress.split('.');
+  if (parts.length !== 4) {
+      throw new Error('Invalid IP address format');
+  }
+  return (parseInt(parts[0]) << 24) |
+         (parseInt(parts[1]) << 16) |
+         (parseInt(parts[2]) << 8) |
+         parseInt(parts[3]);
+}
+
+// -----------------------------------------------------------------------------
+// A function to get a device property using the mac address as key and DB column nakme as parameter
+//  for the value to be returned
 function getDeviceDataByMacAddress(macAddress, dbColumn) {
 
   const sessionDataKey = 'devicesListAll_JSON';  

front/php/templates/language/en_us.json

@@ -72,7 +72,7 @@
         "Device_TableHead_FirstSession" : "First Session",
         "Device_TableHead_LastSession" : "Last Session",
         "Device_TableHead_LastIP" : "Last IP",
-        "Device_TableHead_MAC" : "MAC",
+        "Device_TableHead_MAC" : "Random MAC",
         "Device_TableHead_MAC_full" : "Full MAC",
         "Device_TableHead_LastIPOrder" : "Last IP Order",
         "Device_TableHead_Status" : "Status",
@@ -460,6 +460,14 @@
         "Plugins_Filters_Mac" : "Mac Filter",
         "Plugins_Out_of" : "out of",        
         "Plugins_no_control" : "No form control was found to render this value.",        
+        "Plugins_type_core" : "Core",        
+        "Plugins_type_core_icon" : "fa-solid fa-gem",        
+        "Plugins_type_scanner" : "Scanners",        
+        "Plugins_type_scanner_icon" : "fa-solid fa-magnifying-glass-plus",        
+        "Plugins_type_system" : "System",        
+        "Plugins_type_system_icon" : "fa-solid fa-gear",        
+        "Plugins_type_publisher" : "Publishers",        
+        "Plugins_type_publisher_icon" : "fa-solid fa-comment-dots",        
         "Settings_Metadata_Toggle" : "Show/hide metadata for the given setting.",
         "settings_missing" : "Not all settings loaded, refresh the page! This is probably caused by a high load on the database or app startup sequence.",
         "settings_missing_block" : "You can not save your settings without specifying all setting keys. Refresh the page. This is probably caused by a high load on the database.",
@@ -547,20 +555,7 @@
         "MQTT_icon" : "<i class=\"fa fa-square-rss\"></i>",
         "REPORT_TITLE" : "Report",
         "REPORT_ERROR" : "The page you are looking for is temporarily unavailable, please try again after a few seconds",       
-        "REPORT_MQTT_name" : "Enable MQTT",
+
-        "REPORT_MQTT_description" : "Enable sending notifications via <a target=\"_blank\" href=\"https://www.home-assistant.io/integrations/mqtt/\">MQTT</a> to your Home Assistance instance.",
-        "MQTT_BROKER_name" : "MQTT broker URL",
-        "MQTT_BROKER_description" : "MQTT host URL (do not include <code>http://</code> or <code>https://</code>).",
-        "MQTT_PORT_name" : "MQTT broker port",
-        "MQTT_PORT_description" : "Port number where the broker is listening. Usually <code>1883</code>.",
-        "MQTT_USER_name" : "MQTT user",
-        "MQTT_USER_description" : "User name used to login into your MQTT broker instance.",
-        "MQTT_PASSWORD_name" : "MQTT password",
-        "MQTT_PASSWORD_description" : "Password used to login into your MQTT broker instance.",
-        "MQTT_QOS_name" : "MQTT Quality of Service",
-        "MQTT_QOS_description" : "Quality of service setting for MQTT message sending. <code>0</code> - Low quality to <code>2</code> - High quality. The higher the quality the longer the delay.",
-        "MQTT_DELAY_SEC_name" : "MQTT delay per device",
-        "MQTT_DELAY_SEC_description" : "A little hack - delay adding to the queue in case the process is restarted and previous publish processes aborted (it takes ~<code>2</code>s to update a sensor config on the broker). Tested with <code>2</code>-<code>3</code> seconds of delay. This delay is only applied when devices are created (during the first notification loop). It doesn not affect subsequent scans or notifications.",
         "API_display_name" : "API",
         "API_icon" : "<i class=\"fa fa-arrow-down-up-across-line\"></i>",
         "API_CUSTOM_SQL_name" : "Custom endpoint",

front/plugins/_publisher_apprise/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "_publisher_apprise",
     "unique_prefix": "APPRISE",
+    "plugin_type": "publisher",
     "enabled": true,    
     "data_source":  "script",
     "show_ui": true,

front/plugins/_publisher_email/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "_publisher_email",
     "unique_prefix": "SMTP",
+    "plugin_type": "publisher",
     "enabled": true,
     "data_source": "script",
     "show_ui": true,

front/plugins/_publisher_mqtt/README.md

@@ -0,0 +1,8 @@
+## Overview
+
+- TBC
+
+### Usage
+
+- Go to settings and fill in relevant details.
+

front/plugins/_publisher_mqtt/config.json

@@ -0,0 +1,471 @@
+{
+    "code_name": "_publisher_mqtt",
+    "unique_prefix": "MQTT",
+    "plugin_type": "publisher",
+    "enabled": true,    
+    "data_source":  "script",
+    "show_ui": true,
+    "localized": ["display_name", "description", "icon"],
+	"display_name" : [
+        {
+		  "language_code": "en_us",
+		  "string" : "MQTT publisher"
+		},
+        {
+		  "language_code": "es_es",
+		  "string" : "Habilitar MQTT"
+		}
+    ],
+    "icon":[{
+        "language_code": "en_us",
+        "string" : "<i class=\"fa-solid fa-square-rss\"></i>"
+		}],     
+    "description": [
+        {
+        "language_code": "en_us",
+        "string" : "A plugin to publish a notification via the Apprise gateway."
+		}
+    ],
+    "params" : [],  
+    "database_column_definitions":
+    [          
+        {
+            "column": "Index",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "Plugin",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "Object_PrimaryID",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "url",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				}]
+        },
+        {
+            "column": "Object_SecondaryID",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "DateTimeCreated",
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Sent when"
+				}]
+        },
+        {
+            "column": "DateTimeChanged",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Changed"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Cambiado"					
+                }]
+        },
+        {
+            "column": "Watched_Value1",
+            "css_classes": "col-sm-2",
+            "show": true,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Notification GUID"
+				}]
+        },
+        {
+            "column": "Watched_Value2",
+            "css_classes": "col-sm-8",
+            "show": true,
+            "type": "textarea_readonly",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Result"
+				}]
+        },
+        {
+            "column": "Watched_Value3",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },
+        {
+            "column": "Watched_Value4",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "N/A"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "N/A"					
+                }]
+        },       
+        {
+            "column": "UserData",
+            "css_classes": "col-sm-2",
+            "show": false,
+            "type": "textbox_save",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Comments"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Comentarios"					
+                }]
+        },        
+        {
+            "column": "Status",
+            "css_classes": "col-sm-1",
+            "show": false,
+            "type": "replace",            
+            "default_value":"",
+            "options": [
+                {
+                    "equals": "watched-not-changed",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-square-check'></i><div></div>"
+                },
+                {
+                    "equals": "watched-changed",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-triangle-exclamation'></i></div>"
+                },
+                {
+                    "equals": "new",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
+                },
+                {
+                    "equals": "missing-in-last-scan",
+                    "replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
+                }
+            ],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Status"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Estado"					
+                }]
+        },        
+        {
+            "column": "Extra",
+            "css_classes": "col-sm-3",
+            "show": false,
+            "type": "label",            
+            "default_value":"",
+            "options": [],
+            "localized": ["name"],
+            "name":[{
+                "language_code": "en_us",
+                "string" : "Extra"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Extra"					
+                }]
+        }            
+    ],
+    "settings":[
+        {
+            "function": "RUN",         
+            "events": ["test"],   
+            "type": "text.select",            
+            "default_value":"disabled",
+            "options": ["disabled", "on_notification" ],
+            "localized": ["name", "description"],
+            "name" :[{
+                "language_code": "en_us",
+                "string" : "When to run"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Cuando ejecuta"
+            }],
+            "description": [
+                {
+                "language_code": "en_us",
+                "string" : "Enable sending notifications via <a target=\"_blank\" href=\"https://www.home-assistant.io/integrations/mqtt/\">MQTT</a> to your Home Assistance instance."
+				},
+                {
+                "language_code": "es_es",
+                "string" : "Habilitar el envío de notificaciones a través de <a target=\"_blank\" href=\"https://www.home-assistant.io/integrations/mqtt/\">MQTT</a> a su Home Assistance."
+				}
+            ]
+        },
+        {
+            "function": "CMD",
+            "type": "readonly",
+            "default_value":"python3 /home/pi/pialert/front/plugins/_publisher_mqtt/mqtt.py",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "Command"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Comando"						
+            }],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Command to run"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Comando a ejecutar"	
+            }]
+        },
+        {
+            "function": "RUN_TIMEOUT",
+            "type": "integer",
+            "default_value": 600,
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "Run timeout"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Tiempo de espera de ejecución"				
+				},
+				{
+                "language_code": "de_de",
+                "string" : "Wartezeit"
+            }],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Tiempo máximo en segundos para esperar a que finalice el script. Si se supera este tiempo, el script se cancela."
+            }]
+        },
+        {
+            "function": "BROKER",
+            "type": "text",
+            "default_value": "",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT broker URL"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "URL del broker MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "MQTT host URL (do not include <code>http://</code> or <code>https://</code>)."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "URL del host MQTT (no incluya <code>http://</code> o <code>https://</code>)."
+            }]
+        },
+        {
+            "function": "PORT",
+            "type": "integer",
+            "default_value": 1883,
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT broker port"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Puerto del broker MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Port number where the broker is listening. Usually <code>1883</code>."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Puerto donde escucha el broker MQTT. Normalmente <code>1883</code>."
+                }]
+        },
+        {
+            "function": "USER",
+            "type": "text",
+            "default_value": "",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT user"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Usuario de MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "User name used to login into your MQTT broker instance."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Nombre de usuario utilizado para iniciar sesión en su instancia de agente de MQTT."
+            }]
+        },
+        {
+            "function": "PASSWORD",
+            "type": "password",
+            "default_value": "",
+            "options": [],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT password"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Contraseña de MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Password used to login into your MQTT broker instance."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Contraseña utilizada para iniciar sesión en su instancia de agente de MQTT."
+            }]
+        },
+        {
+            "function": "QOS",
+            "type": "integer.select",
+            "default_value": 0,
+            "options": [0, 1, 2],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT Quality of Service"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Calidad de servicio MQTT"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "Quality of service setting for MQTT message sending. <code>0</code> - Low quality to <code>2</code> - High quality. The higher the quality the longer the delay."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Configuración de calidad de servicio para el envío de mensajes MQTT. <code>0</code>: baja calidad a <code>2</code>: alta calidad. Cuanto mayor sea la calidad, mayor será el retraso."
+            }]
+        },
+        {
+            "function": "DELAY_SEC",
+            "type": "integer",
+            "default_value": 2,
+            "options": [2, 3, 4, 5],
+            "localized": ["name", "description"],
+            "name" : [{
+                "language_code": "en_us",
+                "string" : "MQTT delay per device"
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Retraso de MQTT por dispositivo"				
+				}],
+            "description": [{
+                "language_code": "en_us",
+                "string" : "A little hack - delay adding to the queue in case the process is restarted and previous publish processes aborted (it takes ~<code>2</code>s to update a sensor config on the broker). Tested with <code>2</code>-<code>3</code> seconds of delay. This delay is only applied when devices are created (during the first notification loop). It doesn not affect subsequent scans or notifications."
+				},
+				{
+                "language_code": "es_es",
+                "string" : "Un pequeño truco: retrase la adición a la cola en caso de que el proceso se reinicie y los procesos de publicación anteriores se anulen (se necesitan ~<code>2</code>s para actualizar la configuración de un sensor en el intermediario). Probado con <code>2</code>-<code>3</code> segundos de retraso. Este retraso solo se aplica cuando se crean dispositivos (durante el primer bucle de notificación). No afecta los escaneos o notificaciones posteriores."
+            }]
+        }         
+    ]
+}

front/plugins/_publisher_mqtt/ignore_plugin

@@ -1 +0,0 @@
-This plugin will not be loaded

front/plugins/_publisher_mqtt/mqtt.py

@@ -1,12 +1,48 @@
+#!/usr/bin/env python
 
+import json
+import subprocess
+import argparse
+import os
+import pathlib
+import sys
+from datetime import datetime
 import time
 import re
 from paho.mqtt import client as mqtt_client
 
+
+# Replace these paths with the actual paths to your Pi.Alert directories
+sys.path.extend(["/home/pi/pialert/front/plugins", "/home/pi/pialert/pialert"])
+
+#  PiAlert modules
 import conf
-from logger import mylog
+from plugin_helper import Plugin_Objects
-from database import get_all_devices, get_device_stats
+from logger import mylog, append_line_to_file
-from helper import bytes_to_string, sanitize_string
+from helper import timeNowTZ, noti_obj, get_setting_value, bytes_to_string, sanitize_string
+from notification import Notification_obj
+from database import DB, get_all_devices, get_device_stats
+
+
+CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
+RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
+
+pluginName = 'MQTT'
+
+def main():
+    
+    mylog('verbose', [f'[{pluginName}](publisher) In script'])    
+    
+    # Check if basic config settings supplied
+    if check_config() == False:
+        mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
+        return
+
+    # Create a database connection
+    db = DB()  # instance of class DB
+    db.open()
+
+    mqtt_start(db)
 
 
 
@@ -16,7 +52,7 @@ from helper import bytes_to_string, sanitize_string
 
 #-------------------------------------------------------------------------------
 def check_config():
-        if conf.MQTT_BROKER == '' or conf.MQTT_PORT == '' or conf.MQTT_USER == '' or conf.MQTT_PASSWORD == '':
+        if get_setting_value('MQTT_BROKER') == '' or get_setting_value('MQTT_PORT') == '' or get_setting_value('MQTT_USER') == '' or get_setting_value('MQTT_PASSWORD') == '':
             mylog('none', ['[Check Config] Error: MQTT service not set up correctly. Check your pialert.conf MQTT_* variables.'])
             return False
         else:
@@ -41,14 +77,14 @@ def publish_mqtt(client, topic, message):
         result = client.publish(
                 topic=topic,
                 payload=message,
-                qos=conf.MQTT_QOS,
+                qos=get_setting_value('MQTT_QOS'),
                 retain=True,
             )
 
         status = result[0]
 
         if status != 0:            
-            mylog('minimal', ["Waiting to reconnect to MQTT broker"])
+            mylog('minimal', [f"[{pluginName}] Waiting to reconnect to MQTT broker"])
             time.sleep(0.1) 
     return True
 
@@ -74,6 +110,8 @@ def create_sensor(client, deviceId, deviceName, sensorType, sensorName, icon):
     # check if config already in list and if not, add it, otherwise skip
     is_unique = True
 
+    mylog('minimal', [f"[{pluginName}] Already previously published sensors: {len(conf.mqtt_sensors)}"])
+
     for sensor in conf.mqtt_sensors:
         if sensor.hash == new_sensor_config.hash:
             is_unique = False
@@ -108,7 +146,7 @@ def publish_sensor(client, sensorConf):
     # add the sensor to the global list to keep track of succesfully added sensors
     if publish_mqtt(client, topic, message):        
                                      # hack - delay adding to the queue in case the process is 
-        time.sleep(conf.MQTT_DELAY_SEC)   # restarted and previous publish processes aborted 
+        time.sleep(get_setting_value('MQTT_DELAY_SEC'))   # restarted and previous publish processes aborted 
                                      # (it takes ~2s to update a sensor config on the broker)
         conf.mqtt_sensors.append(sensorConf)
 
@@ -123,18 +161,18 @@ def mqtt_create_client():
     def on_connect(client, userdata, flags, rc):
         
         if rc == 0: 
-            mylog('verbose', ["        Connected to broker"])            
+            mylog('verbose', [f"[{pluginName}]         Connected to broker"])            
             conf.mqtt_connected_to_broker = True     # Signal connection 
         else: 
-            mylog('none', ["        Connection failed"])
+            mylog('none', [f"[{pluginName}]         Connection failed"])
             conf.mqtt_connected_to_broker = False
 
 
     client = mqtt_client.Client('PiAlert')   # Set Connecting Client ID    
-    client.username_pw_set(conf.MQTT_USER, conf.MQTT_PASSWORD)    
+    client.username_pw_set(get_setting_value('MQTT_USER'), get_setting_value('MQTT_PASSWORD'))    
     client.on_connect = on_connect
     client.on_disconnect = on_disconnect
-    client.connect(conf.MQTT_BROKER, conf.MQTT_PORT)
+    client.connect(get_setting_value('MQTT_BROKER'), get_setting_value('MQTT_PORT'))
     client.loop_start() 
 
     return client
@@ -178,9 +216,9 @@ def mqtt_start(db):
     # Get all devices
     devices = get_all_devices(db)
 
-    sec_delay = len(devices) * int(conf.MQTT_DELAY_SEC)*5
+    sec_delay = len(devices) * int(get_setting_value('MQTT_DELAY_SEC'))*5
 
-    mylog('minimal', ["        Estimated delay: ", (sec_delay), 's ', '(', round(sec_delay/60,1) , 'min)' ])
+    mylog('minimal', [f"[{pluginName}]         Estimated delay: ", (sec_delay), 's ', '(', round(sec_delay/60,1) , 'min)' ])
 
     for device in devices:        
 
@@ -243,3 +281,13 @@ def to_binary_sensor(input):
         if bytes_to_string(input) == "1":
             result = "ON"
     return result
+
+
+
+
+#  -------------INIT---------------------
+if __name__ == '__main__':
+    sys.exit(main())
+
+
+

front/plugins/arp_scan/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "arp_scan",
   "unique_prefix": "ARPSCAN",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "CurrentScan",

front/plugins/csv_backup/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "csv_backup",
   "unique_prefix": "CSVBCKP",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",  
   "show_ui": false,

front/plugins/db_cleanup/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "db_cleanup",
   "unique_prefix": "DBCLNP",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",  
   "show_ui": false,

front/plugins/ddns_update/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "ddns_update",
   "unique_prefix": "DDNS",
+  "plugin_type": "system",
   "enabled": true,
   "data_filters": [
     {

front/plugins/dhcp_leases/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "dhcp_leases",
   "unique_prefix": "DHCPLSS",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "data_filters": [

front/plugins/dhcp_servers/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "dhcp_servers",
     "unique_prefix": "DHCPSRVS",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "show_ui": true,

front/plugins/internet_ip/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "internet_ip",
   "unique_prefix": "INTRNT",
+  "plugin_type": "scanner",
   "enabled": true,
   "mapped_to_table": "CurrentScan",
   "data_filters": [

front/plugins/internet_speedtest/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "internet_speedtest",
   "unique_prefix": "INTRSPD",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "show_ui": true,

front/plugins/known_template/config.json

@@ -2,6 +2,7 @@
   "code_name": "known_template",
   "template_type": "database-entry",
   "unique_prefix": "KNWN",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "template", 
   "show_ui": false, 

front/plugins/newdev_template/config.json

@@ -2,6 +2,7 @@
   "code_name": "Devices.new",
   "template_type": "database-entry",
   "unique_prefix": "NEWDEV",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "template",  
   "show_ui": false,

front/plugins/nmap_scan/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "nmap_scan",
     "unique_prefix": "NMAP",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "data_filters": [

front/plugins/pholus_scan/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "pholus_scan",
   "unique_prefix": "PHOLUS",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "Pholus_Scan",

front/plugins/pihole_scan/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "pihole_scan",
   "unique_prefix": "PIHOLE",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "sqlite-db-query",
   "mapped_to_table": "CurrentScan",  

front/plugins/set_password/config.json

@@ -2,6 +2,7 @@
   "code_name": "set_password",
   "template_type": "database-entry",
   "unique_prefix": "SETPWD",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",  
   "show_ui": false,

front/plugins/snmp_discovery/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "snmp_discovery",
     "unique_prefix": "SNMPDSC",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "data_filters": [

front/plugins/undiscoverables/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "undiscoverables",
   "unique_prefix": "UNDIS",
+  "plugin_type": "scanner",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "CurrentScan",

front/plugins/unifi_import/config.json

@@ -1,5 +1,8 @@
 {
     "code_name": "unifi_import",
+    "show_ui": true,
+    "unique_prefix": "UNFIMP",
+    "plugin_type": "scanner",
     "data_filters": [
         {
             "compare_column": "Object_PrimaryID",
@@ -950,7 +953,5 @@
         ],
         "type": "text.select"
         }
-    ],
+    ]    
-    "show_ui": true,
-    "unique_prefix": "UNFIMP"
 }

front/plugins/vendor_update/config.json

@@ -1,6 +1,7 @@
 {
   "code_name": "vendor_update",
   "unique_prefix": "VNDRPDT",
+  "plugin_type": "system",
   "enabled": true,
   "data_source": "script",
   "show_ui": true,

front/plugins/website_monitor/config.json

@@ -1,6 +1,7 @@
 {
     "code_name": "website_monitor",
     "unique_prefix": "WEBMON",
+    "plugin_type": "scanner",
     "enabled": true,    
     "data_source":  "script",
     "show_ui": true,

front/settings.php

@@ -72,8 +72,30 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
       </h1>
       <div class="settingsImported"><?= lang("settings_imported");?> <span id="lastImportedTime"></span></div>      
     </section>
-    <div class="content settingswrap" id='accordion_gen'>
+    <div class="content settingswrap col-sm-12" id="accordion_gen">
-     <!-- PLACEHOLDER -->
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_core");?> (<i class="<?= lang("Plugins_type_core_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="core_content"></div>
+
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_system");?> (<i class="<?= lang("Plugins_type_system_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="system_content"></div>
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_scanner");?> (<i class="<?= lang("Plugins_type_scanner_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="scanner_content"></div>
+
+      <div class ="settings-group col-sm-12" >        
+          <?= lang("Plugins_type_publisher");?> (<i class="<?= lang("Plugins_type_publisher_icon");?>"></i>)        
+      </div>      
+      <div class =" col-sm-12" id="publisher_content"></div>
+
+     
    </div>
    
     <!-- /.content -->
@@ -96,6 +118,25 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
 
 <script>
 
+  // Get plugin type base on prefix
+  function getPluginType(pluginsData, prefix)
+  {
+
+    result = "core_content"
+
+    pluginsData.forEach((plug) => {
+
+      if (plug.unique_prefix == prefix ) {
+        id = plug.plugin_type + "_content"
+        
+        console.log(id)
+        result = plug.plugin_type + "_content";        
+      }
+    });
+
+    return result;
+  }
+
   
   function getData(){
 
@@ -103,18 +144,23 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
         
         settingsData = res["data"];       
 
-        initSettingsPage(settingsData);
+        $.get('api/plugins.json?nocache=' + Date.now(), function(res) {  
+
+          pluginsData = res["data"];  
+
+          initSettingsPage(settingsData, pluginsData);
+        })
     })
   }
 
-  function initSettingsPage(settingsData){
+  function initSettingsPage(settingsData, pluginsData){
 
     const settingGroups = [];
+    const pluginTypes = [];
     const settingKeyOfLists = [];
     // core groups are the ones not generated by plugins
     const settingCoreGroups = ['General'];
 
-
     // Loop through the settingsArray and collect unique settingGroups
     settingsData.forEach((set) => {
       if (!settingGroups.includes(set.Group)) {
@@ -125,7 +171,7 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
 
     console.log(settingGroups);
 
-    let headersHtml = '';    
+    
     let pluginHtml = `
           <div class="row table_row">
             <div class="table_cell bold">
@@ -140,19 +186,11 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
 
     for (const group of settingGroups) {
       let isPlugin = false;
-      let settingGroupTypeHtml = '';
 
-      if (settingCoreGroups.includes(group)) {
+      headerHtml = `<div class="box panel panel-default">
-        settingGroupTypeHtml = '';
-      } else {
-        settingGroupTypeHtml = ' (<i class="fa-regular fa-plug fa-sm"></i>) ';
-        isPlugin = true;
-      }
-
-      headersHtml += `<div class="box panel panel-default">
                   <a data-toggle="collapse" data-parent="#accordion_gen" href="#${group}">
                     <div class="panel-heading">
-                      <h4 class="panel-title">${getString(group+"_icon")} ${getString(group+"_display_name")} ${settingGroupTypeHtml}</h4>
+                      <h4 class="panel-title">${getString(group+"_icon")} ${getString(group+"_display_name")}</h4>
                     </div>
                   </a>
                   <div id="${group}" data-myid="collapsible" class="panel-collapse collapse ${isIn}">
@@ -163,10 +201,11 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
                 </div>
                     `;
       isIn = ' '; // open the first panel only by default on page load
+
+      // generate headers/sections      
+      $('#'+getPluginType(pluginsData, group)).append(headerHtml);
     }  
 
-    // generate headers/sections
-    $('#accordion_gen').html(headersHtml);
 
     // generate panel content
     for (const group of settingGroups) {
@@ -353,7 +392,7 @@ while ($row = $result -> fetchArray (SQLITE3_ASSOC)) {
           const eventsList = createArray(set['Events']);          
 
           if (eventsList.length > 0) {
-            console.log(eventsList)
+            // console.log(eventsList)
             eventsList.forEach(event => {
               eventsHtml += `<span class="input-group-addon pointer"
                 data-myparam="${codeName}"

pialert/conf.py

@@ -81,15 +81,6 @@ NTFY_PASSWORD   = ''
 REPORT_PUSHSAFER    = False 
 PUSHSAFER_TOKEN     = 'ApiKey'
 
-# MQTT
-REPORT_MQTT     = False 
-MQTT_BROKER     = '' 
-MQTT_PORT       = 1883 
-MQTT_USER       = '' 
-MQTT_PASSWORD   = '' 
-MQTT_QOS        = 0 
-MQTT_DELAY_SEC  = 2 
-
 # -------------------------------------------
 # Misc
 # -------------------------------------------

pialert/helper.py

@@ -277,8 +277,7 @@ def get_setting_value(key):
 
     if setting is not None:
 
-        mylog('none', [f'[SETTINGS] setting json:{json.dumps(setting)}'])
+        # mylog('none', [f'[SETTINGS] setting json:{json.dumps(setting)}'])        
-        
 
         set_type  = 'Error: Not handled'
         set_value = 'Error: Not handled'

pialert/initialise.py

@@ -139,15 +139,6 @@ def importConfigs (db):
     conf.REPORT_PUSHSAFER = ccd('REPORT_PUSHSAFER', False , c_d, 'Enable PUSHSAFER', 'boolean', '', 'PUSHSAFER', ['test'])
     conf.PUSHSAFER_TOKEN = ccd('PUSHSAFER_TOKEN', 'ApiKey' , c_d, 'PUSHSAFER token', 'text', '', 'PUSHSAFER')
     
-    # MQTT
-    conf.REPORT_MQTT = ccd('REPORT_MQTT', False , c_d, 'Enable MQTT', 'boolean', '', 'MQTT')
-    conf.MQTT_BROKER = ccd('MQTT_BROKER', '' , c_d, 'MQTT broker', 'text', '', 'MQTT')
-    conf.MQTT_PORT = ccd('MQTT_PORT', 1883 , c_d, 'MQTT broker port', 'integer', '', 'MQTT')
-    conf.MQTT_USER = ccd('MQTT_USER', '' , c_d, 'MQTT user', 'text', '', 'MQTT')
-    conf.MQTT_PASSWORD = ccd('MQTT_PASSWORD', '' , c_d, 'MQTT password', 'password', '', 'MQTT')
-    conf.MQTT_QOS = ccd('MQTT_QOS', 0 , c_d, 'MQTT Quality of Service', 'integer.select', "['0', '1', '2']", 'MQTT')
-    conf.MQTT_DELAY_SEC = ccd('MQTT_DELAY_SEC', 2 , c_d, 'MQTT delay', 'integer.select', "['2', '3', '4', '5']", 'MQTT')
-    
     #  Init timezone in case it changed
     conf.tz = timezone(conf.TIMEZONE) 
 
@@ -281,9 +272,16 @@ def read_config_file(filename):
 
 
 #-------------------------------------------------------------------------------
+# DEPERECATED after 1/1/2024
 replacements = {
     r'\bREPORT_TO\b': 'SMTP_REPORT_TO',
-    r'\bREPORT_FROM\b': 'SMTP_REPORT_FROM'
+    r'\bREPORT_FROM\b': 'SMTP_REPORT_FROM',
+    r'REPORT_MAIL=True': 'SMTP_RUN=\'on_notification\'',
+    r'REPORT_APPRISE=True': 'APPRISE_RUN=\'on_notification\'',
+    r'REPORT_NTFY=True': 'NTFY_RUN=\'on_notification\'',
+    r'REPORT_WEBHOOK=True': 'WEBHOOK_RUN=\'on_notification\'',
+    r'REPORT_PUSHSAFER=True': 'PUSHSAFER_RUN=\'on_notification\'',
+    r'REPORT_MQTT=True': 'MQTT_RUN=\'on_notification\''
 }
 
 def renameSettings(config_file):

pialert/reporting.py

@@ -30,16 +30,12 @@ from publishers.webhook import (check_config as webhook_check_config,
                                 send as send_webhook) 
 from publishers.pushsafer import (check_config as pushsafer_check_config,
                                 send as send_pushsafer) 
-from publishers.mqtt import (check_config as mqtt_check_config,
-                             mqtt_start )
 
 
 #===============================================================================
 # REPORTING
 #===============================================================================
-# create a json of the notifications to provide further integration options (e.g. used in webhook, mqtt notifications)
+# create a json of the notifications to provide further integration options
-
-
 json_final = []