Rename work 🏗

server/README.md

@@ -0,0 +1,31 @@
+# NetAlertX modules
+
+The original pilaert.py code is now moved to this new folder and split into different modules.
+
+| Module | Description |
+|--------|-----------|
+|```__main__.py```| The MAIN program of NetAlertX|
+|```__init__.py```| an empty init file|
+|```README.md```| this readme file|
+|```../front/plugins ```| a folder containing all [plugins](/front/plugins/) that publish notifications or scan for devices|
+|```api.py```| updating the API endpoints with the relevant data. |
+|```appevent.py```| TBC |
+|```const.py```| A place to define the constants for NetAlertX like log path or config path.|
+|```conf.py```| conf.py holds the configuration variables and makes them available for all modules. It is also the <b>workaround</b> for global variables that need to be resolved at some point|
+|```database.py```| This module connects to the DB, makes sure the DB is up to date and defines some standard queries and interfaces. |
+|```device.py```| The device module looks after the devices and saves the scan results into the devices |
+|```flows.py```| TBC |
+|```helper.py```| Helper as the name suggest contains multiple little functions and methods used in many of the other modules and helps keep things clean |
+|```initialise.py```| Initiatlise sets up the environment and makes everything ready to go |
+|```logger.py```| Logger is there the keep all the logs organised and looking identical. |
+|```networscan.py```| Networkscan collects the scan results (maybe to merge with `reporting.py`) |
+|```notification.py```| Creates and handles the notification object and generates ther HTML and text variants of the message |
+|```plugin.py```| This is where the plugins get integrated into the backend of NetAlertX |
+|```plugin_utils.py```| Helper utilities for `plugin.py` |
+|```reporting.py```| Reporting collects the data for the notification reports |
+|```scheduler.py```| All things scheduling |
+
+
+
+
+

server/__init__.py

@@ -0,0 +1 @@
+""" __init__ for NetAlertX """

server/__main__.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+#
+#-------------------------------------------------------------------------------
+#  NetAlertX  v2.70  /  2021-02-01
+#  Open Source Network Guard / WIFI & LAN intrusion detector 
+#
+#  Back module. Network scanner
+#-------------------------------------------------------------------------------
+#  Puche 2021 / 2022+ jokob             jokob@duck.com                GNU GPLv3
+#-------------------------------------------------------------------------------
+
+
+#===============================================================================
+# IMPORTS
+#===============================================================================
+#from __future__ import print_function
+
+import sys
+import time
+import datetime
+import multiprocessing
+
+# Register NetAlertX modules 
+import conf
+from const import *
+from logger import  mylog
+from helper import  filePermissions, timeNowTZ, updateState, get_setting_value
+from api import update_api
+from networkscan import process_scan
+from initialise import importConfigs
+from database import DB
+from reporting import get_notifications
+from notification import Notification_obj
+from plugin import run_plugin_scripts, check_and_run_user_event 
+from device import update_devices_names
+
+
+#===============================================================================
+#===============================================================================
+#                              MAIN
+#===============================================================================
+#===============================================================================
+"""
+main structure of Pi Alert
+
+    Initialise All
+    start Loop forever
+        initialise loop 
+            (re)import config
+            (re)import plugin config
+        run plugins (once)
+        run frontend events
+        update API         
+        run plugins (scheduled)
+        processing scan results
+        run plugins (after Scan)
+        reporting - could be replaced by run flows TODO
+    end loop
+"""
+
+def main ():
+    mylog('none', ['[MAIN] Setting up ...']) # has to be level 'none' as user config not loaded yet
+
+    mylog('none', [f'[conf.tz] Setting up ...{conf.tz}'])
+    
+    # check file permissions and fix if required
+    filePermissions()
+
+    # Open DB once and keep open
+    # Opening / closing DB frequently actually casues more issues
+    db = DB()  # instance of class DB
+    db.open()
+    sql = db.sql  # To-Do replace with the db class
+
+    # Upgrade DB if needed
+    db.upgradeDB()
+
+    #===============================================================================
+    # This is the main loop of NetAlertX 
+    #===============================================================================
+
+    mylog('debug', '[MAIN] Starting loop')
+
+    # Header + init app state
+    updateState("Initializing")    
+
+    while True:
+
+        # re-load user configuration and plugins   
+        importConfigs(db)
+
+        # update time started
+        conf.loop_start_time = timeNowTZ()       
+        
+        loop_start_time = conf.loop_start_time # TODO fix                      
+
+        # Handle plugins executed ONCE
+        if conf.plugins_once_run == False:
+            pluginsState = run_plugin_scripts(db, 'once')  
+            conf.plugins_once_run = True
+
+        # check if there is a front end initiated event which needs to be executed
+        pluginsState = check_and_run_user_event(db, pluginsState)
+
+        # Update API endpoints              
+        update_api(db)
+        
+        # proceed if 1 minute passed
+        if conf.last_scan_run + datetime.timedelta(minutes=1) < conf.loop_start_time :
+
+             # last time any scan or maintenance/upkeep was run
+            conf.last_scan_run = loop_start_time            
+
+            # Header
+            updateState("Process: Start")      
+
+            # Timestamp
+            startTime = loop_start_time
+            startTime = startTime.replace (microsecond=0) 
+
+            # Check if any plugins need to run on schedule
+            pluginsState = run_plugin_scripts(db,'schedule', pluginsState) 
+
+            # determine run/scan type based on passed time
+            # --------------------------------------------
+           
+            # Runs plugin scripts which are set to run every timne after a scans finished            
+            pluginsState = run_plugin_scripts(db,'always_after_scan', pluginsState)
+
+            
+            # process all the scanned data into new devices
+            mylog('debug', [f'[MAIN] processScan: {pluginsState.processScan}'])
+            
+            if pluginsState.processScan == True:   
+                mylog('debug', "[MAIN] start processig scan results")  
+                pluginsState.processScan = False
+                process_scan(db)
+                          
+            # --------
+            # Reporting   
+            # run plugins before notification processing (e.g. Plugins to discover device names)
+            pluginsState = run_plugin_scripts(db, 'before_name_updates', pluginsState)
+
+            # Resolve devices names
+            mylog('debug','[Main] Resolve devices names')
+            update_devices_names(db)             
+            
+            # Check if new devices found
+            sql.execute (sql_new_devices)
+            newDevices = sql.fetchall()
+            db.commitDB()
+            
+            #  new devices were found
+            if len(newDevices) > 0:
+                #  run all plugins registered to be run when new devices are found                    
+                pluginsState = run_plugin_scripts(db, 'on_new_device', pluginsState)                
+
+            # Notification handling
+            # ----------------------------------------
+
+            # send all configured notifications
+            final_json = get_notifications(db)
+
+            # Write the notifications into the DB
+            notification    = Notification_obj(db)
+            notificationObj = notification.create(final_json, "")
+
+            # run all enabled publisher gateways 
+            if notificationObj.HasNotifications:
+                pluginsState = run_plugin_scripts(db, 'on_notification', pluginsState) 
+                notification.setAllProcessed()
+                notification.clearPendingEmailFlag()
+
+                
+            else:
+                mylog('verbose', ['[Notification] No changes to report'])
+
+            # Commit SQL
+            db.commitDB()          
+            
+            # Footer
+            updateState("Process: Wait")
+            mylog('verbose', ['[MAIN] Process: Wait'])            
+        else:
+            # do something  
+            # mylog('verbose', ['[MAIN] Waiting to start next loop'])
+            dummyVariable = 1          
+            
+
+        #loop     
+        time.sleep(5) # wait for N seconds      
+
+
+
+
+
+
+
+
+#===============================================================================
+# BEGIN
+#===============================================================================
+if __name__ == '__main__':
+    mylog('debug', ['[__main__] Welcome to NetAlertX'])
+    sys.exit(main())       

server/api.py

@@ -0,0 +1,92 @@
+import json
+
+
+# Register NetAlertX modules 
+import conf  
+from const import (apiPath, sql_appevents, sql_devices_all, sql_events_pending_alert, sql_settings, sql_plugins_events, sql_plugins_history, sql_plugins_objects,sql_language_strings, sql_notifications_all)
+from logger import mylog
+from helper import write_file
+
+apiEndpoints = []
+
+#===============================================================================
+# API
+#===============================================================================
+def update_api(db, isNotification = False, updateOnlyDataSources = []):
+    mylog('debug', ['[API] Update API starting'])
+    # return
+
+    folder = apiPath 
+
+    # Save plugins    
+    write_file(folder + 'plugins.json'  , json.dumps({"data" : conf.plugins}))  
+
+    #  prepare database tables we want to expose 
+    dataSourcesSQLs = [
+        ["appevents", sql_appevents],        
+        ["devices", sql_devices_all],        
+        ["events_pending_alert", sql_events_pending_alert],
+        ["settings", sql_settings],
+        ["plugins_events", sql_plugins_events],
+        ["plugins_history", sql_plugins_history],
+        ["plugins_objects", sql_plugins_objects],
+        ["plugins_language_strings", sql_language_strings],
+        ["notifications", sql_notifications_all],
+        ["custom_endpoint", conf.API_CUSTOM_SQL],
+    ]
+
+    # Save selected database tables
+    for dsSQL in dataSourcesSQLs:
+
+        if updateOnlyDataSources == [] or dsSQL[0] in updateOnlyDataSources:
+
+            api_endpoint_class(db, dsSQL[1], folder + 'table_' + dsSQL[0] + '.json')
+
+
+#-------------------------------------------------------------------------------
+
+
+class api_endpoint_class:
+    def __init__(self, db, query, path):        
+
+        global apiEndpoints
+        self.db = db
+        self.query = query
+        self.jsonData = db.get_table_as_json(self.query).json
+        self.path = path
+        self.fileName = path.split('/')[-1]
+        self.hash = hash(json.dumps(self.jsonData))
+
+        # check if the endpoint needs to be updated
+        found = False
+        changed = False
+        changedIndex = -1
+        index = 0
+        
+        # search previous endpoint states to check if API needs updating
+        for endpoint in apiEndpoints:
+            # match sql and API endpoint path 
+            if endpoint.query == self.query and endpoint.path == self.path:
+                found = True 
+                if endpoint.hash != self.hash:                    
+                    changed = True
+                    changedIndex = index
+
+            index = index + 1
+        
+        # check if API endpoints have changed or if it's a new one
+        if not found or changed:
+
+            mylog('verbose', [f'[API] Updating {self.fileName} file in /front/api'])
+
+            write_file(self.path, json.dumps(self.jsonData)) 
+            
+            if not found:
+                apiEndpoints.append(self)
+
+            elif changed and changedIndex != -1 and changedIndex < len(apiEndpoints):
+                # update hash
+                apiEndpoints[changedIndex].hash = self.hash
+            else:
+                mylog('minimal', [f'[API] ⚠ ERROR Updating {self.fileName}'])
+            

server/appevent.py

@@ -0,0 +1,391 @@
+import datetime
+import json
+import uuid
+
+# Register NetAlertX modules 
+import conf
+from const import applicationPath, logPath, apiPath, confFileName
+from logger import logResult, mylog, print_log
+from helper import  timeNowTZ
+
+#-------------------------------------------------------------------------------
+# Execution object handling
+#-------------------------------------------------------------------------------
+class AppEvent_obj:
+    def __init__(self, db):
+        self.db = db
+
+        # drop table 
+        self.db.sql.execute("""DROP TABLE IF EXISTS "AppEvents" """)
+
+        # Drop all triggers
+        self.db.sql.execute('DROP TRIGGER IF EXISTS trg_create_device;')
+        self.db.sql.execute('DROP TRIGGER IF EXISTS trg_read_device;')
+        self.db.sql.execute('DROP TRIGGER IF EXISTS trg_update_device;')
+        self.db.sql.execute('DROP TRIGGER IF EXISTS trg_delete_device;')
+
+        self.db.sql.execute('DROP TRIGGER IF EXISTS trg_delete_plugin_object;')
+        self.db.sql.execute('DROP TRIGGER IF EXISTS trg_create_plugin_object;')
+        self.db.sql.execute('DROP TRIGGER IF EXISTS trg_update_plugin_object;')
+
+        # Create AppEvent table if missing
+        self.db.sql.execute("""CREATE TABLE IF NOT EXISTS "AppEvents" (
+            "Index"                 INTEGER,
+            "GUID"                  TEXT UNIQUE,
+            "DateTimeCreated"       TEXT,
+            "ObjectType"            TEXT, -- ObjectType (Plugins, Notifications, Events)
+            "ObjectGUID"            TEXT,
+            "ObjectPlugin"          TEXT,
+            "ObjectPrimaryID"       TEXT,
+            "ObjectSecondaryID"     TEXT,
+            "ObjectForeignKey"      TEXT,
+            "ObjectIndex"           TEXT,            
+            "ObjectIsNew"           BOOLEAN, 
+            "ObjectIsArchived"      BOOLEAN, 
+            "ObjectStatusColumn"    TEXT, -- Status (Notifications, Plugins), eve_EventType (Events)
+            "ObjectStatus"          TEXT, -- new_devices, down_devices, events, new, watched-changed, watched-not-changed, missing-in-last-scan, Device down, New Device, IP Changed, Connected, Disconnected, VOIDED - Disconnected, VOIDED - Connected, <missing event>            
+            "AppEventType"          TEXT, -- "create", "update", "delete" (+TBD)
+            "Helper1"               TEXT,
+            "Helper2"               TEXT,
+            "Helper3"               TEXT,
+            "Extra"                 TEXT,            
+            PRIMARY KEY("Index" AUTOINCREMENT)
+        );
+        """)
+
+        # Generate a GUID
+        sql_generateGuid = '''
+                        lower(
+                            hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-' || '4' || 
+                            substr(hex( randomblob(2)), 2) || '-' || 
+                            substr('AB89', 1 + (abs(random()) % 4) , 1)  ||
+                            substr(hex(randomblob(2)), 2) || '-' || 
+                            hex(randomblob(6))
+                        )
+                    '''
+
+        # -------------
+        # Device events
+
+        sql_devices_mappedColumns = '''
+                    "GUID",
+                    "DateTimeCreated",
+                    "ObjectType",
+                    "ObjectPrimaryID",
+                    "ObjectSecondaryID",
+                    "ObjectStatus",
+                    "ObjectStatusColumn",
+                    "ObjectIsNew",
+                    "ObjectIsArchived",
+                    "ObjectForeignKey",
+                    "AppEventType"
+        '''
+
+        # Trigger for create event
+        self.db.sql.execute(f'''
+            CREATE TRIGGER IF NOT EXISTS "trg_create_device"
+            AFTER INSERT ON "Devices"
+            BEGIN
+                INSERT INTO "AppEvents" (
+                    {sql_devices_mappedColumns}
+                )
+                VALUES (                    
+                    {sql_generateGuid},
+                    DATETIME('now'),
+                    'Devices',
+                    NEW.dev_MAC,
+                    NEW.dev_LastIP,
+                    CASE WHEN NEW.dev_PresentLastScan = 1 THEN 'online' ELSE 'offline' END,
+                    'dev_PresentLastScan',
+                    NEW.dev_NewDevice,
+                    NEW.dev_Archived,
+                    NEW.dev_MAC,
+                    'create'
+                );
+            END;
+        ''')
+
+        # 🔴 This would generate too many events, disabled for now
+        # # Trigger for read event
+        # self.db.sql.execute('''
+        #     TODO
+        # ''')
+
+        # Trigger for update event
+        self.db.sql.execute(f'''
+            CREATE TRIGGER IF NOT EXISTS "trg_update_device"
+            AFTER UPDATE ON "Devices"
+            BEGIN
+                INSERT INTO "AppEvents" (
+                    {sql_devices_mappedColumns}
+                )
+                VALUES (                    
+                    {sql_generateGuid},
+                    DATETIME('now'),
+                    'Devices',
+                    NEW.dev_MAC,
+                    NEW.dev_LastIP,
+                    CASE WHEN NEW.dev_PresentLastScan = 1 THEN 'online' ELSE 'offline' END,
+                    'dev_PresentLastScan',
+                    NEW.dev_NewDevice,
+                    NEW.dev_Archived,
+                    NEW.dev_MAC,
+                    'update'
+                );
+            END;
+        ''')
+
+        # Trigger for delete event
+        self.db.sql.execute(f'''
+            CREATE TRIGGER IF NOT EXISTS "trg_delete_device"
+            AFTER DELETE ON "Devices"
+            BEGIN
+                INSERT INTO "AppEvents" (
+                    {sql_devices_mappedColumns}
+                )
+                VALUES (                    
+                    {sql_generateGuid},
+                    DATETIME('now'),
+                    'Devices',
+                    OLD.dev_MAC,
+                    OLD.dev_LastIP,
+                    CASE WHEN OLD.dev_PresentLastScan = 1 THEN 'online' ELSE 'offline' END,
+                    'dev_PresentLastScan',
+                    OLD.dev_NewDevice,
+                    OLD.dev_Archived,
+                    OLD.dev_MAC,
+                    'delete'
+                );
+            END;
+        ''')
+
+
+        # -------------
+        # Plugins_Objects events
+
+        sql_plugins_objects_mappedColumns = '''
+                    "GUID",
+                    "DateTimeCreated",
+                    "ObjectType",                    
+                    "ObjectPlugin",
+                    "ObjectPrimaryID",
+                    "ObjectSecondaryID",
+                    "ObjectForeignKey",
+                    "ObjectStatusColumn",
+                    "ObjectStatus",
+                    "AppEventType"
+        '''
+
+        # Create trigger for update event on Plugins_Objects
+        self.db.sql.execute(f'''
+            CREATE TRIGGER IF NOT EXISTS trg_update_plugin_object
+            AFTER UPDATE ON Plugins_Objects
+            BEGIN
+                INSERT INTO AppEvents (
+                   {sql_plugins_objects_mappedColumns}
+                )
+                VALUES (
+                    {sql_generateGuid},
+                    DATETIME('now'),
+                    'Plugins_Objects',                    
+                    NEW.Plugin,
+                    NEW.Object_PrimaryID,
+                    NEW.Object_SecondaryID,
+                    NEW.ForeignKey,
+                    'Status',
+                    NEW.Status,
+                    'update'
+                );
+            END;
+        ''')
+
+        # Create trigger for CREATE event on Plugins_Objects
+        self.db.sql.execute(f'''
+            CREATE TRIGGER IF NOT EXISTS trg_create_plugin_object
+            AFTER INSERT ON Plugins_Objects
+            BEGIN
+                INSERT INTO AppEvents (
+                {sql_plugins_objects_mappedColumns}
+                )
+                VALUES (
+                    {sql_generateGuid},
+                    DATETIME('now'),
+                    'Plugins_Objects',
+                    NEW.Plugin,
+                    NEW.Object_PrimaryID,
+                    NEW.Object_SecondaryID,
+                    NEW.ForeignKey,
+                    'Status',
+                    NEW.Status,
+                    'create'
+                );
+            END;
+        ''')
+
+        # Create trigger for DELETE event on Plugins_Objects
+        self.db.sql.execute(f'''
+            CREATE TRIGGER IF NOT EXISTS trg_delete_plugin_object
+            AFTER DELETE ON Plugins_Objects
+            BEGIN
+                INSERT INTO AppEvents (
+                {sql_plugins_objects_mappedColumns}
+                )
+                VALUES (
+                    {sql_generateGuid},
+                    DATETIME('now'),
+                    'Plugins_Objects',
+                    OLD.Plugin,
+                    OLD.Object_PrimaryID,
+                    OLD.Object_SecondaryID,
+                    OLD.ForeignKey,
+                    'Status',
+                    OLD.Status,
+                    'delete'
+                );
+            END;
+        ''')
+
+        self.save()
+
+    # -------------------------------------------------------------------------------
+    # -------------------------------------------------------------------------------
+    # below code is unused
+    # -------------------------------------------------------------------------------
+
+    # Create a new DB entry if new notifications are available, otherwise skip
+    def create(self, Extra="", **kwargs):
+        # Check if nothing to report, end
+        if not any(kwargs.values()):
+            return False
+
+        # Continue and save into DB if notifications are available
+        self.GUID = str(uuid.uuid4())
+        self.DateTimeCreated = timeNowTZ()
+        self.ObjectType = "Plugins"  # Modify ObjectType as needed
+
+        # Optional parameters
+        self.ObjectGUID         = kwargs.get("ObjectGUID", "")
+        self.ObjectPlugin       = kwargs.get("ObjectPlugin", "")
+        self.ObjectMAC          = kwargs.get("ObjectMAC", "")
+        self.ObjectIP           = kwargs.get("ObjectIP", "")
+        self.ObjectPrimaryID    = kwargs.get("ObjectPrimaryID", "")
+        self.ObjectSecondaryID  = kwargs.get("ObjectSecondaryID", "")
+        self.ObjectForeignKey   = kwargs.get("ObjectForeignKey", "")
+        self.ObjectIndex        = kwargs.get("ObjectIndex", "")
+        self.ObjectRowID        = kwargs.get("ObjectRowID", "")
+        self.ObjectStatusColumn = kwargs.get("ObjectStatusColumn", "")
+        self.ObjectStatus       = kwargs.get("ObjectStatus", "")
+
+        self.AppEventStatus = "new"  # Modify AppEventStatus as needed
+        self.Extra = Extra
+
+        self.upsert()
+
+        return True
+
+    def upsert(self):
+        self.db.sql.execute("""
+            INSERT OR REPLACE INTO AppEvents (
+                "GUID", 
+                "DateTimeCreated", 
+                "ObjectType", 
+                "ObjectGUID", 
+                "ObjectPlugin", 
+                "ObjectMAC", 
+                "ObjectIP", 
+                "ObjectPrimaryID", 
+                "ObjectSecondaryID", 
+                "ObjectForeignKey", 
+                "ObjectIndex", 
+                "ObjectRowID", 
+                "ObjectStatusColumn", 
+                "ObjectStatus", 
+                "AppEventStatus", 
+                "Extra"
+            )
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (
+            self.GUID, 
+            self.DateTimeCreated, 
+            self.ObjectType, 
+            self.ObjectGUID, 
+            self.ObjectPlugin, 
+            self.ObjectMAC, 
+            self.ObjectIP, 
+            self.ObjectPrimaryID, 
+            self.ObjectSecondaryID, 
+            self.ObjectForeignKey, 
+            self.ObjectIndex, 
+            self.ObjectRowID, 
+            self.ObjectStatusColumn, 
+            self.ObjectStatus, 
+            self.AppEventStatus, 
+            self.Extra
+        ))
+
+        self.save()
+
+    def save(self):
+        # Commit changes
+        self.db.commitDB()
+
+
+def getPluginObject(**kwargs):
+
+    # Check if nothing, end
+    if not any(kwargs.values()):
+            return None
+
+    # Optional parameters
+    GUID          = kwargs.get("GUID", "")
+    Plugin        = kwargs.get("Plugin", "")
+    MAC           = kwargs.get("MAC", "")
+    IP            = kwargs.get("IP", "")
+    PrimaryID     = kwargs.get("PrimaryID", "")
+    SecondaryID   = kwargs.get("SecondaryID", "")
+    ForeignKey    = kwargs.get("ForeignKey", "")
+    Index         = kwargs.get("Index", "")
+    RowID         = kwargs.get("RowID", "")
+
+    # we need the plugin
+    if Plugin == "":
+        return None
+
+    plugins_objects = apiPath + 'table_plugins_objects.json'
+
+    try:
+        with open(plugins_objects, 'r') as json_file:
+
+            data = json.load(json_file)
+
+            for item in data.get("data",[]):
+                if item.get("Index") == Index:
+                    return item
+
+            for item in data.get("data",[]):
+                if item.get("ObjectPrimaryID") == PrimaryID and item.get("ObjectSecondaryID") == SecondaryID:                    
+                    return item
+            
+            for item in data.get("data",[]):
+                if item.get("ObjectPrimaryID") == MAC and item.get("ObjectSecondaryID") == IP:
+                    return item
+
+            for item in data.get("data",[]):
+                if item.get("ObjectPrimaryID") == PrimaryID and item.get("ObjectSecondaryID") == IP:
+                    return item
+
+            for item in data.get("data",[]):
+                if item.get("ObjectPrimaryID") == MAC and item.get("ObjectSecondaryID") == IP:
+                    return item
+                
+
+            mylog('debug', [f'[{module_name}] ⚠ ERROR - Object not found - GUID:{GUID} | Plugin:{Plugin} | MAC:{MAC} | IP:{IP} | PrimaryID:{PrimaryID} | SecondaryID:{SecondaryID} | ForeignKey:{ForeignKey} | Index:{Index} | RowID:{RowID} '])  
+
+            return None
+
+    except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
+        # Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format
+        mylog('none', [f'[{module_name}] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {plugins_objects}'])                
+
+        return None
+

server/conf.py

@@ -0,0 +1,49 @@
+""" config related functions for NetAlertX """
+
+# TODO: Create and manage this as part of an app_state class object
+#===============================================================================
+
+# These are global variables, not config items and should not exist !
+mySettings = []
+mySettingsSQLsafe = []
+cycle = 1
+userSubnets = []
+mySchedules = [] # bad solution for global - TO-DO
+plugins = []  # bad solution for global - TO-DO
+tz = ''
+
+# modified time of the most recently imported config file
+# set to a small value to force import at first run
+lastImportedConfFile = 1.1 
+
+plugins_once_run = False
+newVersionAvailable = False
+time_started = ''
+startTime = ''
+last_scan_run = ''
+last_version_check = ''
+arpscan_devices = []
+
+# ACTUAL CONFIGRATION ITEMS set to defaults
+
+# -------------------------------------------
+# General
+# -------------------------------------------
+SCAN_SUBNETS    = ['192.168.1.0/24 --interface=eth1', '192.168.1.0/24 --interface=eth0']   
+LOG_LEVEL       = 'verbose' 
+TIMEZONE        = 'Europe/Berlin'
+UI_LANG         = 'English' 
+UI_PRESENCE       = ['online', 'offline', 'archived']  
+UI_MY_DEVICES     = ['online', 'offline', 'archived', 'new', 'down']  
+UI_NOT_RANDOM_MAC = []
+PIALERT_WEB_PROTECTION  = False 
+PIALERT_WEB_PASSWORD    = '8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92' 
+DAYS_TO_KEEP_EVENTS     = 90 
+REPORT_DASHBOARD_URL    = 'http://netalertx/' 
+
+# -------------------------------------------
+# Misc
+# -------------------------------------------
+
+# API     
+API_CUSTOM_SQL  = 'SELECT * FROM Devices WHERE dev_PresentLastScan = 0'

server/const.py

@@ -0,0 +1,48 @@
+""" CONSTANTS for NetAlertX """
+
+#===============================================================================
+# PATHS
+#===============================================================================
+applicationPath = '/app'
+dbFileName      = 'app.db'
+confFileName    = 'app.conf'
+confPath        = "/config/" + confFileName
+
+dbPath          = '/db/' + dbFileName
+
+
+pluginsPath  = applicationPath + '/front/plugins'
+logPath      = applicationPath + '/front/log'
+apiPath      = applicationPath + '/front/api/'
+fullConfPath = applicationPath + confPath
+fullDbPath   = applicationPath + dbPath
+vendorsPath = '/usr/share/arp-scan/ieee-oui.txt'
+
+
+
+
+#===============================================================================
+# SQL queries
+#===============================================================================
+sql_devices_all = """select rowid, * from Devices"""
+sql_appevents = """select * from AppEvents"""
+sql_devices_stats =  """SELECT Online_Devices as online, Down_Devices as down, All_Devices as 'all', Archived_Devices as archived, 
+                        (select count(*) from Devices a where dev_NewDevice = 1 ) as new, 
+                        (select count(*) from Devices a where dev_Name = '(unknown)' or dev_Name = '(name not found)' ) as unknown 
+                        from Online_History order by Scan_Date desc limit 1"""
+sql_events_pending_alert = "SELECT  * FROM Events where eve_PendingAlertEmail is not 0"
+sql_settings = "SELECT  * FROM Settings"
+sql_plugins_objects = "SELECT  * FROM Plugins_Objects"
+sql_language_strings = "SELECT  * FROM Plugins_Language_Strings"
+sql_notifications_all = "SELECT  * FROM Notifications"
+sql_plugins_events = "SELECT  * FROM Plugins_Events"
+sql_plugins_history = "SELECT  * FROM Plugins_History ORDER BY DateTimeChanged DESC"
+sql_new_devices = """SELECT * FROM ( 
+                        SELECT eve_IP as dev_LastIP, eve_MAC as dev_MAC 
+                        FROM Events_Devices
+                        WHERE eve_PendingAlertEmail = 1
+                        AND eve_EventType = 'New Device'
+                        ORDER BY eve_DateTime ) t1
+                        LEFT JOIN 
+                        ( SELECT dev_Name, dev_MAC as dev_MAC_t2 FROM Devices) t2 
+                        ON t1.dev_MAC = t2.dev_MAC_t2"""

server/database.py

@@ -0,0 +1,507 @@
+""" all things database to support NetAlertX """
+
+import sqlite3
+import base64
+
+# Register NetAlertX modules 
+from const import fullDbPath, sql_devices_stats, sql_devices_all
+
+from logger import mylog
+from helper import json_obj, initOrSetParam, row_to_json, timeNowTZ #, updateState
+from appevent import AppEvent_obj
+
+
+
+class DB():
+    """
+    DB Class to provide the basic database interactions.
+    Open / Commit / Close / read / write
+    """
+
+    def __init__(self):
+        self.sql = None
+        self.sql_connection = None
+
+    #-------------------------------------------------------------------------------
+    def open (self):
+        # Check if DB is open
+        if self.sql_connection != None :
+            mylog('debug','openDB: database already open')
+            return
+
+        mylog('none', '[Database] Opening DB' )
+        # Open DB and Cursor
+        try:
+            self.sql_connection = sqlite3.connect (fullDbPath, isolation_level=None)
+            self.sql_connection.execute('pragma journal_mode=wal') #
+            self.sql_connection.text_factory = str
+            self.sql_connection.row_factory = sqlite3.Row
+            self.sql = self.sql_connection.cursor()
+        except sqlite3.Error as e:
+            mylog('none',[ '[Database] - Open DB Error: ', e])
+
+
+    #-------------------------------------------------------------------------------
+    def commitDB (self):
+        if self.sql_connection == None :
+            mylog('debug','commitDB: database is not open')
+            return False
+
+        # Commit changes to DB
+        self.sql_connection.commit()
+        return True
+
+    #-------------------------------------------------------------------------------
+    def rollbackDB(self):
+        if self.sql_connection:
+            self.sql_connection.rollback()
+
+    #-------------------------------------------------------------------------------    
+    def get_sql_array(self, query):
+        if self.sql_connection == None :
+            mylog('debug','getQueryArray: database is not open')
+            return
+
+        self.sql.execute(query)
+        rows = self.sql.fetchall()
+        #self.commitDB()
+
+        #  convert result into list of lists
+        arr = []
+        for row in rows:
+            r_temp = []
+            for column in row:
+                r_temp.append(column)
+            arr.append(r_temp)
+
+        return arr
+
+
+    #-------------------------------------------------------------------------------
+    def upgradeDB(self):
+        """
+        Check the current tables in the DB and upgrade them if neccessary
+        """
+
+        # indicates, if Online_History table is available
+        onlineHistoryAvailable = self.sql.execute("""
+            SELECT name FROM sqlite_master WHERE type='table'
+                AND name='Online_History';
+        """).fetchall() != []
+
+        # Check if it is incompatible (Check if table has all required columns)
+        isIncompatible = False
+
+        if onlineHistoryAvailable :
+          isIncompatible = self.sql.execute ("""
+                SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Online_History') WHERE name='Archived_Devices'
+            """).fetchone()[0] == 0
+
+        # Drop table if available, but incompatible
+        if onlineHistoryAvailable and isIncompatible:
+          mylog('none','[upgradeDB] Table is incompatible, Dropping the Online_History table')
+          self.sql.execute("DROP TABLE Online_History;")
+          onlineHistoryAvailable = False
+
+        if onlineHistoryAvailable == False :
+          self.sql.execute("""
+          CREATE TABLE "Online_History" (
+            "Index"	INTEGER,
+            "Scan_Date"	TEXT,
+            "Online_Devices"	INTEGER,
+            "Down_Devices"	INTEGER,
+            "All_Devices"	INTEGER,
+            "Archived_Devices" INTEGER,
+            PRIMARY KEY("Index" AUTOINCREMENT)
+          );
+          """)
+
+        # -------------------------------------------------------------------------
+        # Alter Devices table       
+        # -------------------------------------------------------------------------
+        # dev_Network_Node_MAC_ADDR column
+        dev_Network_Node_MAC_ADDR_missing = self.sql.execute ("""
+            SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_MAC_ADDR'
+          """).fetchone()[0] == 0
+
+        if dev_Network_Node_MAC_ADDR_missing :
+          mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_MAC_ADDR to the Devices table"])
+          self.sql.execute("""
+            ALTER TABLE "Devices" ADD "dev_Network_Node_MAC_ADDR" TEXT
+          """)
+
+        # dev_Network_Node_port column
+        dev_Network_Node_port_missing = self.sql.execute ("""
+            SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Network_Node_port'
+          """).fetchone()[0] == 0
+
+        if dev_Network_Node_port_missing :
+          mylog('verbose', ["[upgradeDB] Adding dev_Network_Node_port to the Devices table"])
+          self.sql.execute("""
+            ALTER TABLE "Devices" ADD "dev_Network_Node_port" INTEGER
+          """)
+
+        # dev_Icon column
+        dev_Icon_missing = self.sql.execute ("""
+            SELECT COUNT(*) AS CNTREC FROM pragma_table_info('Devices') WHERE name='dev_Icon'
+          """).fetchone()[0] == 0
+
+        if dev_Icon_missing :
+          mylog('verbose', ["[upgradeDB] Adding dev_Icon to the Devices table"])
+          self.sql.execute("""
+            ALTER TABLE "Devices" ADD "dev_Icon" TEXT
+          """)
+
+
+        # -------------------------------------------------------------------------
+        # Settings table setup
+        # -------------------------------------------------------------------------
+
+        
+        # Re-creating Settings table
+        mylog('verbose', ["[upgradeDB] Re-creating Settings table"])
+
+        self.sql.execute(""" DROP TABLE IF EXISTS Settings;""")
+        self.sql.execute("""
+            CREATE TABLE "Settings" (
+            "Code_Name"	    TEXT,
+            "Display_Name"	TEXT,
+            "Description"	TEXT,
+            "Type"          TEXT,
+            "Options"       TEXT,
+            "RegEx"         TEXT,
+            "Value"	        TEXT,
+            "Group"	        TEXT,
+            "Events"	    TEXT
+            );
+            """)
+
+
+        # -------------------------------------------------------------------------
+        # Pholus_Scan table setup
+        # -------------------------------------------------------------------------
+
+        # Create Pholus_Scan table if missing
+        mylog('verbose', ["[upgradeDB] Re-creating Pholus_Scan table"])
+        self.sql.execute("""CREATE TABLE IF NOT EXISTS "Pholus_Scan" (
+            "Index"	          INTEGER,
+            "Info"	          TEXT,
+            "Time"	          TEXT,
+            "MAC"	          TEXT,
+            "IP_v4_or_v6"	  TEXT,
+            "Record_Type"	  TEXT,
+            "Value"           TEXT,
+            "Extra"           TEXT,
+            PRIMARY KEY("Index" AUTOINCREMENT)
+        );
+        """)
+
+
+        # -------------------------------------------------------------------------
+        # Parameters table setup
+        # -------------------------------------------------------------------------
+
+        # Re-creating Parameters table
+        mylog('verbose', ["[upgradeDB] Re-creating Parameters table"])
+        self.sql.execute("DROP TABLE Parameters;")
+
+        self.sql.execute("""
+          CREATE TABLE "Parameters" (
+            "par_ID" TEXT PRIMARY KEY,
+            "par_Value"	TEXT
+          );
+          """)        
+
+        # -------------------------------------------------------------------------
+        # Nmap_Scan table setup DEPRECATED after 9/9/2024
+        # -------------------------------------------------------------------------
+
+        # indicates, if Nmap_Scan table is available
+        nmapScanMissing = self.sql.execute("""
+        SELECT name FROM sqlite_master WHERE type='table'
+        AND name='Nmap_Scan';
+        """).fetchone() == None
+
+        if nmapScanMissing == False:
+            # move data into the PLugins_Objects table
+            self.sql.execute("""INSERT INTO Plugins_Objects (
+                                    Plugin,
+                                    Object_PrimaryID,
+                                    Object_SecondaryID,
+  									DateTimeCreated,
+									DateTimeChanged,
+                                    Watched_Value1,
+                                    Watched_Value2,
+                                    Watched_Value3,
+                                    Watched_Value4,
+                                    Status,
+                                    Extra,
+                                    UserData,
+                                    ForeignKey
+                                )
+                                SELECT
+                                    'NMAP' AS Plugin,
+                                    MAC AS Object_PrimaryID,
+                                    Port AS Object_SecondaryID,
+									Time AS DateTimeCreated,
+									DATETIME('now') AS DateTimeChanged,
+                                    State AS Watched_Value1,
+                                    Service AS Watched_Value2,
+                                    '' AS Watched_Value3,
+                                    '' AS Watched_Value4,
+                                    'watched-not-changed' AS Status,
+                                    Extra AS Extra,
+                                    Extra AS UserData,
+                                    MAC AS ForeignKey
+                                FROM Nmap_Scan;""")
+
+            # Delete the Nmap_Scan table
+            self.sql.execute("DROP TABLE Nmap_Scan;")
+            nmapScanMissing = True
+
+        # -------------------------------------------------------------------------
+        # Nmap_Scan table setup DEPRECATED after 9/9/2024 cleanup above
+        # -------------------------------------------------------------------------
+
+        # -------------------------------------------------------------------------
+        # Icon format migration table setup DEPRECATED after 9/9/2024 cleanup below
+        # -------------------------------------------------------------------------
+
+        sql_Icons = """ UPDATE Devices SET dev_Icon = '<i class="fa fa-' || dev_Icon || '"></i>'
+                WHERE dev_Icon NOT LIKE '<i class="fa fa-%'
+                AND dev_Icon NOT LIKE '<svg%' 
+                AND dev_Icon NOT LIKE 'PGkg%' 
+                AND dev_Icon NOT LIKE 'PHN%' 
+                AND dev_Icon NOT IN ('', 'null')
+                 """
+        self.sql.execute(sql_Icons)
+        self.commitDB()      
+
+        # Base64 conversion
+
+        self.sql.execute("SELECT dev_MAC, dev_Icon FROM Devices WHERE dev_Icon like '<%' ")
+        icons = self.sql.fetchall()
+
+
+        # Loop through the icons, encode them, and update the database
+        for icon_tuple in icons:
+            icon = icon_tuple[1]
+            
+            # Encode the icon as base64
+            encoded_icon = base64.b64encode(icon.encode('utf-8')).decode('ascii')
+            # Update the database with the encoded icon
+            sql_update = f"""
+                UPDATE Devices
+                SET dev_Icon = '{encoded_icon}'
+                WHERE dev_MAC = '{icon_tuple[0]}'
+            """
+
+            self.sql.execute(sql_update)
+
+        # -------------------------------------------------------------------------
+        # Icon format migration table setup DEPRECATED after 9/9/2024 cleanup above
+        # -------------------------------------------------------------------------
+
+        # -------------------------------------------------------------------------
+        # Plugins tables setup
+        # -------------------------------------------------------------------------
+
+        # Plugin state
+        sql_Plugins_Objects = """ CREATE TABLE IF NOT EXISTS Plugins_Objects(
+                                    "Index"	          INTEGER,
+                                    Plugin TEXT NOT NULL,
+                                    Object_PrimaryID TEXT NOT NULL,
+                                    Object_SecondaryID TEXT NOT NULL,
+                                    DateTimeCreated TEXT NOT NULL,
+                                    DateTimeChanged TEXT NOT NULL,
+                                    Watched_Value1 TEXT NOT NULL,
+                                    Watched_Value2 TEXT NOT NULL,
+                                    Watched_Value3 TEXT NOT NULL,
+                                    Watched_Value4 TEXT NOT NULL,
+                                    Status TEXT NOT NULL,
+                                    Extra TEXT NOT NULL,
+                                    UserData TEXT NOT NULL,
+                                    ForeignKey TEXT NOT NULL,
+                                    PRIMARY KEY("Index" AUTOINCREMENT)
+                        ); """
+        self.sql.execute(sql_Plugins_Objects)
+
+        # Plugin execution results
+        sql_Plugins_Events = """ CREATE TABLE IF NOT EXISTS Plugins_Events(
+                                    "Index"	          INTEGER,
+                                    Plugin TEXT NOT NULL,
+                                    Object_PrimaryID TEXT NOT NULL,
+                                    Object_SecondaryID TEXT NOT NULL,
+                                    DateTimeCreated TEXT NOT NULL,
+                                    DateTimeChanged TEXT NOT NULL,
+                                    Watched_Value1 TEXT NOT NULL,
+                                    Watched_Value2 TEXT NOT NULL,
+                                    Watched_Value3 TEXT NOT NULL,
+                                    Watched_Value4 TEXT NOT NULL,
+                                    Status TEXT NOT NULL,
+                                    Extra TEXT NOT NULL,
+                                    UserData TEXT NOT NULL,
+                                    ForeignKey TEXT NOT NULL,
+                                    PRIMARY KEY("Index" AUTOINCREMENT)
+                        ); """
+        self.sql.execute(sql_Plugins_Events)
+
+        # Plugin execution history
+        sql_Plugins_History = """ CREATE TABLE IF NOT EXISTS Plugins_History(
+                                    "Index"	          INTEGER,
+                                    Plugin TEXT NOT NULL,
+                                    Object_PrimaryID TEXT NOT NULL,
+                                    Object_SecondaryID TEXT NOT NULL,
+                                    DateTimeCreated TEXT NOT NULL,
+                                    DateTimeChanged TEXT NOT NULL,
+                                    Watched_Value1 TEXT NOT NULL,
+                                    Watched_Value2 TEXT NOT NULL,
+                                    Watched_Value3 TEXT NOT NULL,
+                                    Watched_Value4 TEXT NOT NULL,
+                                    Status TEXT NOT NULL,
+                                    Extra TEXT NOT NULL,
+                                    UserData TEXT NOT NULL,
+                                    ForeignKey TEXT NOT NULL,
+                                    PRIMARY KEY("Index" AUTOINCREMENT)
+                        ); """
+        self.sql.execute(sql_Plugins_History)
+
+        # -------------------------------------------------------------------------
+        # Plugins_Language_Strings table setup
+        # -------------------------------------------------------------------------
+
+        # Dynamically generated language strings
+        self.sql.execute("DROP TABLE IF EXISTS Plugins_Language_Strings;")
+        self.sql.execute(""" CREATE TABLE IF NOT EXISTS Plugins_Language_Strings(
+                                "Index"	          INTEGER,
+                                Language_Code TEXT NOT NULL,
+                                String_Key TEXT NOT NULL,
+                                String_Value TEXT NOT NULL,
+                                Extra TEXT NOT NULL,
+                                PRIMARY KEY("Index" AUTOINCREMENT)
+                        ); """)
+
+        self.commitDB()        
+
+        # -------------------------------------------------------------------------
+        # CurrentScan table setup
+        # -------------------------------------------------------------------------
+        
+        # indicates, if CurrentScan table is available
+        self.sql.execute("DROP TABLE IF EXISTS CurrentScan;")
+        self.sql.execute(""" CREATE TABLE CurrentScan (                                
+                                cur_MAC STRING(50) NOT NULL COLLATE NOCASE,
+                                cur_IP STRING(50) NOT NULL COLLATE NOCASE,
+                                cur_Vendor STRING(250),
+                                cur_ScanMethod STRING(10),
+                                cur_Name STRING(250),
+                                cur_LastQuery STRING(250),
+                                cur_DateTime STRING(250)
+                            );
+                        """)
+
+        # Init the AppEvent database table
+        AppEvent_obj(self)
+
+        # -------------------------------------------------------------------------
+        #  DELETING OBSOLETE TABLES - to remove with updated db file after 9/9/2024
+        # -------------------------------------------------------------------------        
+
+        # Deletes obsolete ScanCycles
+        self.sql.execute(""" DROP TABLE IF EXISTS ScanCycles;""")
+        self.sql.execute(""" DROP TABLE IF EXISTS DHCP_Leases;""")
+        self.sql.execute(""" DROP TABLE IF EXISTS PiHole_Network;""")
+
+        self.commitDB()
+
+        # -------------------------------------------------------------------------
+        #  DELETING OBSOLETE TABLES - to remove with updated db file after 9/9/2024
+        # -------------------------------------------------------------------------
+
+
+    #-------------------------------------------------------------------------------
+    def get_table_as_json(self, sqlQuery):
+
+        # mylog('debug',[ '[Database] - get_table_as_json - Query: ', sqlQuery])
+        try:
+            self.sql.execute(sqlQuery)
+            columnNames = list(map(lambda x: x[0], self.sql.description))
+            rows = self.sql.fetchall()
+        except sqlite3.Error as e:
+            mylog('none',[ '[Database] - SQL ERROR: ', e])
+            return json_obj({}, []) # return empty object
+
+        result = {"data":[]}
+        for row in rows:
+            tmp = row_to_json(columnNames, row)
+            result["data"].append(tmp)
+
+        # mylog('debug',[ '[Database] - get_table_as_json - returning ', len(rows), " rows with columns: ", columnNames])
+        return json_obj(result, columnNames)
+
+    #-------------------------------------------------------------------------------
+    # referece from here: https://codereview.stackexchange.com/questions/241043/interface-class-for-sqlite-databases
+    #-------------------------------------------------------------------------------
+    def read(self, query, *args):
+        """check the query and arguments are aligned and are read only"""
+        # mylog('debug',[ '[Database] - Read All: SELECT Query: ', query, " params: ", args])
+        try:
+            assert query.count('?') == len(args)
+            assert query.upper().strip().startswith('SELECT')
+            self.sql.execute(query, args)
+            rows = self.sql.fetchall()
+            return rows
+        except AssertionError:
+            mylog('none',[ '[Database] - ERROR: inconsistent query and/or arguments.', query, " params: ", args])
+        except sqlite3.Error as e:
+            mylog('none',[ '[Database] - SQL ERROR: ', e])
+        return None
+
+    def read_one(self, query, *args):
+        """ 
+        call read() with the same arguments but only returns the first row.
+        should only be used when there is a single row result expected
+        """
+
+        mylog('debug',[ '[Database] - Read One: ', query, " params: ", args])
+        rows = self.read(query, *args)
+        if len(rows) == 1:
+            return rows[0]
+                
+        if len(rows) > 1: 
+            mylog('none',[ '[Database] - Warning!: query returns multiple rows, only first row is passed on!', query, " params: ", args])
+            return rows[0]
+        # empty result set
+        return None
+
+
+
+#-------------------------------------------------------------------------------
+def get_device_stats(db):
+    # columns = ["online","down","all","archived","new","unknown"]
+    return db.read_one(sql_devices_stats)
+#-------------------------------------------------------------------------------
+def get_all_devices(db):
+    return db.read(sql_devices_all)
+
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+def insertOnlineHistory(db):
+    sql = db.sql #TO-DO
+    startTime = timeNowTZ()
+    # Add to History
+
+    History_All = db.read("SELECT * FROM Devices")
+    History_All_Devices  = len(History_All)
+
+    History_Archived = db.read("SELECT * FROM Devices WHERE dev_Archived = 1")
+    History_Archived_Devices  = len(History_Archived)
+
+    History_Online = db.read("SELECT * FROM Devices WHERE dev_PresentLastScan = 1")
+    History_Online_Devices  = len(History_Online)
+    History_Offline_Devices = History_All_Devices - History_Archived_Devices - History_Online_Devices
+
+    sql.execute ("INSERT INTO Online_History (Scan_Date, Online_Devices, Down_Devices, All_Devices, Archived_Devices) "+
+                 "VALUES ( ?, ?, ?, ?, ?)", (startTime, History_Online_Devices, History_Offline_Devices, History_All_Devices, History_Archived_Devices ) )
+    db.commitDB()

server/device.py

@@ -0,0 +1,435 @@
+
+import subprocess
+
+import conf
+import re
+from helper import timeNowTZ, get_setting, get_setting_value, list_to_where, resolve_device_name_dig, resolve_device_name_pholus, get_device_name_nslookup, check_IP_format
+from logger import mylog, print_log
+from const import vendorsPath
+
+#-------------------------------------------------------------------------------
+# Device object handling (WIP)
+#-------------------------------------------------------------------------------
+class Device_obj:
+    def __init__(self, db):
+        self.db = db
+
+    # Get all
+    def getAll(self):
+        self.db.sql.execute("""
+            SELECT * FROM Devices
+        """)
+        return self.db.sql.fetchall()
+    
+    # Get all with unknown names
+    def getUnknown(self):
+        self.db.sql.execute("""
+            SELECT * FROM Devices WHERE dev_Name in ("(unknown)", "(name not found)", "" )
+        """)
+        return self.db.sql.fetchall()
+
+
+#-------------------------------------------------------------------------------
+def save_scanned_devices (db):
+    sql = db.sql #TO-DO
+
+
+    # Add Local MAC of default local interface
+    local_mac_cmd = ["/sbin/ifconfig `ip -o route get 1 | sed 's/^.*dev \\([^ ]*\\).*$/\\1/;q'` | grep ether | awk '{print $2}'"]
+    local_mac = subprocess.Popen (local_mac_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().strip()
+
+    local_ip_cmd = ["ip -o route get 1 | sed 's/^.*src \\([^ ]*\\).*$/\\1/;q'"]
+    local_ip = subprocess.Popen (local_ip_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0].decode().strip()
+
+    mylog('debug', ['[Save Devices] Saving this IP into the CurrentScan table:', local_ip])
+
+    if check_IP_format(local_ip) == '':
+        local_ip = '0.0.0.0'
+
+    # Proceed if variable contains valid MAC
+    if check_mac_or_internet(local_mac):
+        # Check if local mac has been detected with other methods
+        sql.execute (f"SELECT COUNT(*) FROM CurrentScan WHERE cur_MAC = '{local_mac}'")
+        if sql.fetchone()[0] == 0 :
+            sql.execute (f"""INSERT INTO CurrentScan (cur_MAC, cur_IP, cur_Vendor, cur_ScanMethod) VALUES ( '{local_mac}', '{local_ip}', Null, 'local_MAC') """)
+
+#-------------------------------------------------------------------------------
+def print_scan_stats(db):
+    sql = db.sql # TO-DO
+
+    query = """
+    SELECT
+        (SELECT COUNT(*) FROM CurrentScan) AS devices_detected,
+        (SELECT COUNT(*) FROM CurrentScan WHERE NOT EXISTS (SELECT 1 FROM Devices WHERE dev_MAC = cur_MAC)) AS new_devices,
+        (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown != 0 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS down_alerts,
+        (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown != 0 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS new_down_alerts,
+        (SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 0) AS new_connections,
+        (SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS disconnections,
+        (SELECT COUNT(*) FROM Devices, CurrentScan WHERE dev_MAC = cur_MAC AND dev_LastIP <> cur_IP) AS ip_changes,
+        cur_ScanMethod,
+        COUNT(*) AS scan_method_count
+    FROM CurrentScan
+    GROUP BY cur_ScanMethod
+    """
+
+    sql.execute(query)
+    stats = sql.fetchall()
+
+    mylog('verbose', f'[Scan Stats] Devices Detected.......: {stats[0]["devices_detected"]}')
+    mylog('verbose', f'[Scan Stats] New Devices............: {stats[0]["new_devices"]}')
+    mylog('verbose', f'[Scan Stats] Down Alerts............: {stats[0]["down_alerts"]}')
+    mylog('verbose', f'[Scan Stats] New Down Alerts........: {stats[0]["new_down_alerts"]}')
+    mylog('verbose', f'[Scan Stats] New Connections........: {stats[0]["new_connections"]}')
+    mylog('verbose', f'[Scan Stats] Disconnections.........: {stats[0]["disconnections"]}')
+    mylog('verbose', f'[Scan Stats] IP Changes.............: {stats[0]["ip_changes"]}')
+
+    if str(stats[0]["new_devices"]) != '0':
+        mylog('debug', f'   ================ DEVICES table content  ================')
+        sql.execute('select * from Devices')
+        rows = sql.fetchall()
+        for row in rows:
+            row_dict = dict(row)
+            mylog('debug', f'    {row_dict}')
+        
+        mylog('debug', f'   ================ CurrentScan table content  ================')
+        sql.execute('select * from CurrentScan')
+        rows = sql.fetchall()
+        for row in rows:
+            row_dict = dict(row)
+            mylog('debug', f'    {row_dict}')
+        
+        mylog('debug', f'   ================ Events table content where eve_PendingAlertEmail = 1  ================')
+        sql.execute('select * from Events where eve_PendingAlertEmail = 1')
+        rows = sql.fetchall()
+        for row in rows:
+            row_dict = dict(row)
+            mylog('debug', f'    {row_dict}')
+
+        mylog('debug', f'   ================ Events table COUNT  ================')
+        sql.execute('select count(*) from Events')
+        rows = sql.fetchall()
+        for row in rows:
+            row_dict = dict(row)
+            mylog('debug', f'    {row_dict}')
+        
+
+    mylog('verbose', '[Scan Stats] Scan Method Statistics:')
+    for row in stats:
+        if row["cur_ScanMethod"] is not None:
+            mylog('verbose', f'    {row["cur_ScanMethod"]}: {row["scan_method_count"]}')
+
+
+#-------------------------------------------------------------------------------
+def create_new_devices (db):
+    sql = db.sql # TO-DO
+    startTime = timeNowTZ()
+
+    # Insert events for new devices from CurrentScan
+    mylog('debug','[New Devices] New devices - 1 Events')
+
+    query = f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+                        eve_EventType, eve_AdditionalInfo,
+                        eve_PendingAlertEmail)
+                    SELECT cur_MAC, cur_IP, '{startTime}', 'New Device', cur_Vendor, 1
+                    FROM CurrentScan
+                    WHERE NOT EXISTS (SELECT 1 FROM Devices
+                                      WHERE dev_MAC = cur_MAC) 
+                            {list_to_where('OR', 'cur_MAC', 'NOT LIKE', get_setting_value('NEWDEV_ignored_MACs'))}
+                            {list_to_where('OR', 'cur_IP', 'NOT LIKE', get_setting_value('NEWDEV_ignored_IPs'))}
+                """ 
+
+    
+    mylog('debug',f'[New Devices] Query: {query}')
+    
+    sql.execute(query)
+
+    mylog('debug',f'[New Devices] Insert Connection into session table')
+
+    sql.execute (f"""INSERT INTO Sessions (ses_MAC, ses_IP, ses_EventTypeConnection, ses_DateTimeConnection,
+                        ses_EventTypeDisconnection, ses_DateTimeDisconnection, ses_StillConnected, ses_AdditionalInfo)
+                    SELECT cur_MAC, cur_IP,'Connected','{startTime}', NULL , NULL ,1, cur_Vendor
+                    FROM CurrentScan 
+                    WHERE NOT EXISTS (SELECT 1 FROM Sessions
+                                      WHERE ses_MAC = cur_MAC) 
+                            {list_to_where('OR', 'cur_MAC', 'NOT LIKE', get_setting_value('NEWDEV_ignored_MACs'))}
+                            {list_to_where('OR', 'cur_IP', 'NOT LIKE', get_setting_value('NEWDEV_ignored_IPs'))}
+                    """)
+                    
+    # Create new devices from CurrentScan
+    mylog('debug','[New Devices] 2 Create devices')
+
+    # default New Device values preparation
+    newDevColumns  =   """dev_AlertEvents, 
+                          dev_AlertDeviceDown, 
+                          dev_PresentLastScan, 
+                          dev_Archived, 
+                          dev_NewDevice, 
+                          dev_SkipRepeated, 
+                          dev_ScanCycle, 
+                          dev_Owner, 
+                          dev_DeviceType, 
+                          dev_Favorite, 
+                          dev_Group, 
+                          dev_Comments, 
+                          dev_LogEvents, 
+                          dev_Location, 
+                          dev_Network_Node_MAC_ADDR, 
+                          dev_Icon"""
+
+    newDevDefaults =  f"""{get_setting_value('NEWDEV_dev_AlertEvents')}, 
+                          {get_setting_value('NEWDEV_dev_AlertDeviceDown')}, 
+                          {get_setting_value('NEWDEV_dev_PresentLastScan')}, 
+                          {get_setting_value('NEWDEV_dev_Archived')}, 
+                          {get_setting_value('NEWDEV_dev_NewDevice')}, 
+                          {get_setting_value('NEWDEV_dev_SkipRepeated')}, 
+                          {get_setting_value('NEWDEV_dev_ScanCycle')}, 
+                          '{get_setting_value('NEWDEV_dev_Owner')}', 
+                          '{get_setting_value('NEWDEV_dev_DeviceType')}',
+                          {get_setting_value('NEWDEV_dev_Favorite')}, 
+                          '{get_setting_value('NEWDEV_dev_Group')}', 
+                          '{get_setting_value('NEWDEV_dev_Comments')}', 
+                          {get_setting_value('NEWDEV_dev_LogEvents')}, 
+                          '{get_setting_value('NEWDEV_dev_Location')}',  
+                          '{get_setting_value('NEWDEV_dev_Network_Node_MAC_ADDR')}',  
+                          '{get_setting_value('NEWDEV_dev_Icon')}'
+                    """
+    
+    # Bulk-inserting devices from the CurrentScan table as new devices in the table Devices ... 
+    # ... with new device defaults and ignoring specidfied IPs and MACs)
+    sqlQuery = f"""INSERT OR IGNORE INTO Devices (dev_MAC, dev_name, dev_Vendor,
+                        dev_LastIP, dev_FirstConnection, dev_LastConnection,
+                        {newDevColumns})
+                    SELECT cur_MAC, 
+                        CASE WHEN LENGTH(TRIM(cur_Name)) > 0 THEN cur_Name
+                                ELSE '(unknown)' END,
+                        cur_Vendor, cur_IP, ?, ?,
+                        {newDevDefaults}
+                    FROM CurrentScan
+                        WHERE 1=1
+                        {list_to_where('OR', 'cur_MAC', 'NOT LIKE', get_setting_value('NEWDEV_ignored_MACs'))}
+                        {list_to_where('OR', 'cur_IP', 'NOT LIKE', get_setting_value('NEWDEV_ignored_IPs'))}
+                """
+
+
+    mylog('debug',f'[New Devices] Create devices SQL: {sqlQuery}')
+
+    sql.execute (sqlQuery, (startTime, startTime) ) 
+
+    
+    mylog('debug','[New Devices] New Devices end')
+    db.commitDB()
+
+
+#-------------------------------------------------------------------------------
+def update_devices_data_from_scan (db):
+    sql = db.sql #TO-DO    
+    startTime = timeNowTZ()
+
+    # Update Last Connection
+    mylog('debug', '[Update Devices] 1 Last Connection')
+    sql.execute(f"""UPDATE Devices SET dev_LastConnection = '{startTime}',
+                        dev_PresentLastScan = 1
+                    WHERE dev_PresentLastScan = 0
+                      AND EXISTS (SELECT 1 FROM CurrentScan 
+                                  WHERE dev_MAC = cur_MAC) """)
+
+    # Clean no active devices
+    mylog('debug', '[Update Devices] 2 Clean no active devices')
+    sql.execute("""UPDATE Devices SET dev_PresentLastScan = 0
+                    WHERE NOT EXISTS (SELECT 1 FROM CurrentScan 
+                                      WHERE dev_MAC = cur_MAC) """)
+
+    # Update IP 
+    mylog('debug', '[Update Devices] - 3 LastIP ')
+    sql.execute("""UPDATE Devices
+                    SET dev_LastIP = (SELECT cur_IP FROM CurrentScan
+                                      WHERE dev_MAC = cur_MAC)
+                    WHERE EXISTS (SELECT 1 FROM CurrentScan
+                                  WHERE dev_MAC = cur_MAC) """)
+
+    # Update only devices with empty or NULL vendors
+    mylog('debug', '[Update Devices] - 3 Vendor')
+    sql.execute("""UPDATE Devices
+                    SET dev_Vendor = (
+                        SELECT cur_Vendor
+                        FROM CurrentScan
+                        WHERE dev_MAC = cur_MAC
+                    )
+                    WHERE 
+                        (dev_Vendor = "" OR dev_Vendor IS NULL)
+                        AND EXISTS (
+                            SELECT 1
+                            FROM CurrentScan
+                            WHERE dev_MAC = cur_MAC
+                        )""")
+
+    # Update (unknown) or (name not found) Names if available
+    mylog('debug','[Update Devices] - 4 Unknown Name')
+    sql.execute ("""UPDATE Devices
+                    SET dev_NAME = (SELECT cur_Name FROM CurrentScan
+                                    WHERE cur_MAC = dev_MAC)
+                    WHERE (dev_Name in ("(unknown)", "(name not found)", "" )
+                           OR dev_Name IS NULL)
+                      AND EXISTS (SELECT 1 FROM CurrentScan
+                                  WHERE cur_MAC = dev_MAC
+                                    AND cur_Name IS NOT NULL
+                                    AND cur_Name IS NOT 'null'
+                                    AND cur_Name <> '') """)
+
+    recordsToUpdate = []
+    query = """SELECT * FROM Devices
+               WHERE dev_Vendor = '(unknown)' OR dev_Vendor =''
+                  OR dev_Vendor IS NULL"""
+
+    for device in sql.execute (query) :
+        vendor = query_MAC_vendor (device['dev_MAC'])
+        if vendor != -1 and vendor != -2 :
+            recordsToUpdate.append ([vendor, device['dev_MAC']])
+
+    sql.executemany ("UPDATE Devices SET dev_Vendor = ? WHERE dev_MAC = ? ",
+        recordsToUpdate )
+    
+    
+    mylog('debug','[Update Devices] Update devices end')
+
+#-------------------------------------------------------------------------------
+def update_devices_names (db):
+    sql = db.sql #TO-DO
+    # Initialize variables
+    recordsToUpdate = []
+    recordsNotFound = []
+
+    nameNotFound = "(name not found)"
+
+    ignored = 0
+    notFound = 0
+
+    foundDig = 0
+    foundNsLookup = 0
+    foundPholus = 0
+
+    # Gen unknown devices
+    sql.execute ("SELECT * FROM Devices WHERE dev_Name IN ('(unknown)','', '(name not found)') AND dev_LastIP <> '-'")
+    unknownDevices = sql.fetchall() 
+    db.commitDB()
+
+    # skip checks if no unknown devices
+    if len(unknownDevices) == 0:
+        return
+
+    # Devices without name
+    mylog('verbose', f'[Update Device Name] Trying to resolve devices without name. Unknown devices count: {len(unknownDevices)}')
+
+    # get names from Pholus scan 
+    sql.execute ('SELECT * FROM Pholus_Scan where "Record_Type"="Answer"')    
+    pholusResults = list(sql.fetchall())        
+    db.commitDB()
+
+    # Number of entries from previous Pholus scans
+    mylog('verbose', ['[Update Device Name] Pholus entries from prev scans: ', len(pholusResults)])
+
+
+    for device in unknownDevices:
+        newName = nameNotFound
+        
+        # Resolve device name with DiG
+        newName = resolve_device_name_dig (device['dev_MAC'], device['dev_LastIP'])
+        
+        # count
+        if newName != nameNotFound:
+            foundDig += 1
+
+        # Resolve device name with NSLOOKUP plugin data
+        if newName == nameNotFound:
+            newName = get_device_name_nslookup(db, device['dev_MAC'], device['dev_LastIP'])
+
+            if newName != nameNotFound:
+               foundNsLookup += 1
+
+        # Resolve with Pholus 
+        if newName == nameNotFound:
+
+            # Try MAC matching
+            newName =  resolve_device_name_pholus (device['dev_MAC'], device['dev_LastIP'], pholusResults, nameNotFound, False)
+            # Try IP matching 
+            if newName == nameNotFound:
+                newName =  resolve_device_name_pholus (device['dev_MAC'], device['dev_LastIP'], pholusResults, nameNotFound, True)
+
+            # count
+            if newName != nameNotFound:
+                foundPholus += 1
+        
+        # if still not found update name so we can distinguish the devices where we tried already
+        if newName == nameNotFound :
+
+            notFound += 1
+
+            # if dev_Name is the same as what we will change it to, take no action
+            # this mitigates a race condition which would overwrite a users edits that occured since the select earlier
+            if device['dev_Name'] != nameNotFound:
+                recordsNotFound.append (["(name not found)", device['dev_MAC']])          
+        else:
+            # name was found with DiG or Pholus
+            recordsToUpdate.append ([newName, device['dev_MAC']])
+
+    # Print log            
+    mylog('verbose', ['[Update Device Name] Names Found (DiG/NSLOOKUP/Pholus): ', len(recordsToUpdate), " (",foundDig,"/",foundNsLookup,"/",foundPholus ,")"] )                 
+    mylog('verbose', ['[Update Device Name] Names Not Found         : ', notFound] )    
+     
+    # update not found devices with (name not found) 
+    sql.executemany ("UPDATE Devices SET dev_Name = ? WHERE dev_MAC = ? ", recordsNotFound )
+    # update names of devices which we were bale to resolve
+    sql.executemany ("UPDATE Devices SET dev_Name = ? WHERE dev_MAC = ? ", recordsToUpdate )
+    db.commitDB()
+
+#-------------------------------------------------------------------------------
+# Check if the variable contains a valid MAC address or "Internet"
+def check_mac_or_internet(input_str):
+    # Regular expression pattern for matching a MAC address
+    mac_pattern = r'^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$'
+
+    if input_str.lower() == 'internet':
+        return True
+    elif re.match(mac_pattern, input_str):
+        return True
+    else:
+        return False
+
+
+#===============================================================================
+# Lookup unknown vendors on devices
+#===============================================================================
+
+#-------------------------------------------------------------------------------
+def query_MAC_vendor (pMAC):
+
+    pMACstr = str(pMAC)
+    
+    # Check MAC parameter
+    mac = pMACstr.replace (':','')
+    if len(pMACstr) != 17 or len(mac) != 12 :
+        return -2 # return -2 if ignored MAC
+
+    # Search vendor in HW Vendors DB
+    mac_start_string6 = mac[0:6]    
+    mac_start_string9 = mac[0:9]    
+
+    try:
+        with open(vendorsPath, 'r') as f:
+            for line in f:
+                if line.startswith(mac_start_string6):                    
+                    parts = line.split(' ', 1)
+                    if len(parts) > 1:
+                        vendor = parts[1].strip()
+                        mylog('debug', [f"[Vendor Check] Found '{vendor}' for '{pMAC}' in {vendorsPath}"])
+                        return vendor
+                    else:
+                        mylog('debug', [f'[Vendor Check] ⚠ ERROR: Match found, but line could not be processed: "{line}"'])
+                        return -1
+
+
+        return -1  # MAC address not found in the database
+    except FileNotFoundError:
+        mylog('none', [f"[Vendor Check] ⚠ ERROR: Vendors file {vendorsPath} not found."])
+        return -1
+

server/flows.py

@@ -0,0 +1,31 @@
+import json
+
+def update_value(json_data, object_path, key, value, target_property, desired_value):
+    # Helper function to traverse the JSON structure and get the target object
+    def traverse(obj, path):
+        keys = path.split(".")
+        for key in keys:
+            if isinstance(obj, list):
+                key = int(key)
+            obj = obj[key]
+        return obj
+
+    # Helper function to update the target property with the desired value
+    def update(obj, path, key, value, target_property, desired_value):
+        keys = path.split(".")
+        for i, key in enumerate(keys):
+            if isinstance(obj, list):
+                key = int(key)
+            # Check if we have reached the desired object
+            if i == len(keys) - 1 and obj[key][key] == value:
+                # Update the target property with the desired value
+                obj[key][target_property] = desired_value
+            else:
+                obj = obj[key]
+        return obj
+
+    # Get the target object based on the object path
+    target_obj = traverse(json_data, object_path)
+    # Update the value in the target object
+    updated_obj = update(json_data, object_path, key, value, target_property, desired_value)
+    return updated_obj

server/helper.py

@@ -0,0 +1,787 @@
+""" Colection of generic functions to support NetAlertX """
+
+import io
+import sys
+import datetime
+# from datetime import strptime 
+import os
+import re
+import subprocess
+import pytz
+from pytz import timezone
+import json
+import time
+from pathlib import Path
+import requests
+
+import conf
+from const import *
+from logger import mylog, logResult
+
+# Register NetAlertX directories
+INSTALL_PATH="/app"
+
+#-------------------------------------------------------------------------------
+# DateTime
+#-------------------------------------------------------------------------------
+# Get the current time in the current TimeZone
+def timeNowTZ():
+    if conf.tz:
+        return datetime.datetime.now(conf.tz).replace(microsecond=0)
+    else:
+        return datetime.datetime.now().replace(microsecond=0)
+    # if isinstance(conf.TIMEZONE, str):
+    #     tz = pytz.timezone(conf.TIMEZONE)
+    # else:
+    #     tz = conf.TIMEZONE
+
+    # return datetime.datetime.now(tz).replace(microsecond=0)
+
+def timeNow():
+    return datetime.datetime.now().replace(microsecond=0)
+
+def get_timezone_offset():    
+    now = datetime.datetime.now(conf.tz)
+    offset_hours = now.utcoffset().total_seconds() / 3600        
+    offset_formatted =  "{:+03d}:{:02d}".format(int(offset_hours), int((offset_hours % 1) * 60))
+    return offset_formatted
+
+
+#-------------------------------------------------------------------------------
+# App state
+#-------------------------------------------------------------------------------
+# A class to manage the application state and to provide a frontend accessible API point
+class app_state_class:
+    def __init__(self, currentState, settingsSaved=None, settingsImported=None, showSpinner=False):
+        # json file containing the state to communicate with the frontend
+        stateFile = apiPath + '/app_state.json'
+
+        # if currentState == 'Initializing':
+        #     checkNewVersion(False)
+
+        # Update self
+        self.currentState = currentState
+        self.lastUpdated = str(timeNowTZ())
+
+        # Check if the file exists and init values
+        if os.path.exists(stateFile):
+            with open(stateFile, 'r') as json_file:
+                previousState               = json.load(json_file)
+                self.settingsSaved          = previousState.get("settingsSaved", 0)
+                self.settingsImported       = previousState.get("settingsImported", 0)
+                self.showSpinner            = previousState.get("showSpinner", False)
+                self.isNewVersion           = previousState.get("isNewVersion", False)
+                self.isNewVersionChecked    = previousState.get("isNewVersionChecked", 0)
+        else:
+            self.settingsSaved          = 0
+            self.settingsImported       = 0
+            self.showSpinner            = False
+            self.isNewVersion           = checkNewVersion()
+            self.isNewVersionChecked    = int(timeNow().timestamp())
+
+        # Overwrite with provided parameters if supplied
+        if settingsSaved is not None:
+            self.settingsSaved = settingsSaved
+        if settingsImported is not None:
+            self.settingsImported = settingsImported
+        if showSpinner is not None:
+            self.showSpinner = showSpinner
+
+        # check for new version every hour and if currently not running new version
+        if self.isNewVersion is False and self.isNewVersionChecked + 3600 < int(timeNow().timestamp()):
+            self.isNewVersion           = checkNewVersion()
+            self.isNewVersionChecked    = int(timeNow().timestamp())
+
+        # Update .json file
+        with open(stateFile, 'w') as json_file:
+            json.dump(self, json_file, cls=AppStateEncoder, indent=4)
+
+         
+    def isSet(self):  
+
+        result = False       
+
+        if self.currentState != "":
+            result = True
+
+        return result
+
+
+#-------------------------------------------------------------------------------
+# method to update the state
+def updateState(newState, settingsSaved = None, settingsImported = None, showSpinner = False):
+
+    state = app_state_class(newState, settingsSaved, settingsImported, showSpinner)
+
+
+#-------------------------------------------------------------------------------
+def updateSubnets(scan_subnets):
+    subnets = []
+
+    # multiple interfaces
+    if type(scan_subnets) is list:
+        for interface in scan_subnets :
+            subnets.append(interface)
+    # one interface only
+    else:
+        subnets.append(scan_subnets)
+
+    return subnets
+
+
+
+#-------------------------------------------------------------------------------
+# File system permission handling
+#-------------------------------------------------------------------------------
+# check RW access of DB and config file
+def checkPermissionsOK():
+    #global confR_access, confW_access, dbR_access, dbW_access
+
+    confR_access = (os.access(fullConfPath, os.R_OK))
+    confW_access = (os.access(fullConfPath, os.W_OK))
+    dbR_access = (os.access(fullDbPath, os.R_OK))
+    dbW_access = (os.access(fullDbPath, os.W_OK))
+
+    mylog('none', ['\n'])
+    mylog('none', ['The container restarted (started). If this is unexpected check https://bit.ly/NetAlertX_debug for troubleshooting tips.'])
+    mylog('none', ['\n'])
+    mylog('none', ['Permissions check (All should be True)'])
+    mylog('none', ['------------------------------------------------'])
+    mylog('none', [ "  " , confPath ,     " | " , " READ  | " , confR_access])
+    mylog('none', [ "  " , confPath ,     " | " , " WRITE | " , confW_access])
+    mylog('none', [ "  " , dbPath , "       | " , " READ  | " , dbR_access])
+    mylog('none', [ "  " , dbPath , "       | " , " WRITE | " , dbW_access])
+    mylog('none', ['------------------------------------------------'])
+
+    #return dbR_access and dbW_access and confR_access and confW_access
+    return (confR_access, dbR_access)
+#-------------------------------------------------------------------------------
+def fixPermissions():
+    # Try fixing access rights if needed
+    chmodCommands = []
+
+    chmodCommands.append(['sudo', 'chmod', 'a+rw', '-R', fullDbPath])
+    chmodCommands.append(['sudo', 'chmod', 'a+rw', '-R', fullConfPath])
+
+    for com in chmodCommands:
+        # Execute command
+        mylog('none', ["[Setup] Attempting to fix permissions."])
+        try:
+            # try runnning a subprocess
+            result = subprocess.check_output (com, universal_newlines=True)
+        except subprocess.CalledProcessError as e:
+            # An error occured, handle it
+            mylog('none', ["[Setup] Fix Failed. Execute this command manually inside of the container: ", ' '.join(com)])
+            mylog('none', [e.output])
+
+
+#-------------------------------------------------------------------------------
+def initialiseFile(pathToCheck, defaultFile):
+    # if file not readable (missing?) try to copy over the backed-up (default) one
+    if str(os.access(pathToCheck, os.R_OK)) == "False":
+        mylog('none', ["[Setup] ("+ pathToCheck +") file is not readable or missing. Trying to copy over the default one."])
+        try:
+            # try runnning a subprocess
+            p = subprocess.Popen(["cp", defaultFile , pathToCheck], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+            stdout, stderr = p.communicate()
+
+            if str(os.access(pathToCheck, os.R_OK)) == "False":
+                mylog('none', ["[Setup] ⚠ ERROR copying ("+defaultFile+") to ("+pathToCheck+"). Make sure the app has Read & Write access to the parent directory."])
+            else:
+                mylog('none', ["[Setup] ("+defaultFile+") copied over successfully to ("+pathToCheck+")."])
+
+            # write stdout and stderr into .log files for debugging if needed
+            logResult (stdout, stderr)  # TO-DO should be changed to mylog
+
+        except subprocess.CalledProcessError as e:
+            # An error occured, handle it
+            mylog('none', ["[Setup] ⚠ ERROR copying ("+defaultFile+"). Make sure the app has Read & Write access to " + pathToCheck])
+            mylog('none', [e.output])
+
+#-------------------------------------------------------------------------------
+def filePermissions():
+    # check and initialize .conf
+    (confR_access, dbR_access) = checkPermissionsOK() # Initial check
+
+    if confR_access == False:
+        initialiseFile(fullConfPath, f"{INSTALL_PATH}/back/app.conf" )
+
+    # check and initialize .db
+    if dbR_access == False:
+        initialiseFile(fullDbPath, f"{INSTALL_PATH}/back/app.db")
+
+    # last attempt
+    fixPermissions()
+
+
+#-------------------------------------------------------------------------------
+# File manipulation methods
+#-------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
+def get_file_content(path):
+
+    f = open(path, 'r')
+    content = f.read()
+    f.close()
+
+    return content
+
+#-------------------------------------------------------------------------------
+def write_file(pPath, pText):
+    # Convert pText to a string if it's a dictionary
+    if isinstance(pText, dict):
+        pText = json.dumps(pText)
+
+    # Convert pText to a string if it's a list
+    if isinstance(pText, list):
+        for item in pText:
+            write_file(pPath, item)
+
+    else:
+        # Write the text using the correct Python version
+        if sys.version_info < (3, 0):
+            file = io.open(pPath, mode='w', encoding='utf-8')
+            file.write(pText.decode('unicode_escape'))
+            file.close()
+        else:
+            file = open(pPath, 'w', encoding='utf-8')
+            if pText is None:
+                pText = ""
+            file.write(pText)
+            file.close()
+
+#-------------------------------------------------------------------------------
+# Setting methods
+#-------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
+#  Return whole setting touple
+def get_setting(key):
+
+    settingsFile = apiPath + 'table_settings.json'
+
+    try:
+        with open(settingsFile, 'r') as json_file:
+
+            data = json.load(json_file)
+
+            for item in data.get("data",[]):
+                if item.get("Code_Name") == key:
+                    return item
+
+            mylog('debug', [f'[Settings] ⚠ ERROR - setting_missing - Setting not found for key: {key} in file {settingsFile}'])  
+
+            return None
+
+    except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
+        # Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format
+        mylog('none', [f'[Settings] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {settingsFile}'])                
+
+        return None
+
+
+
+#-------------------------------------------------------------------------------
+# Settings
+#-------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
+#  Return setting value
+def get_setting_value(key):
+    
+    setting = get_setting(key)
+
+    value = ''
+
+    if setting is not None:
+
+        # mylog('none', [f'[SETTINGS] setting json:{json.dumps(setting)}'])        
+
+        set_type  = 'Error: Not handled'
+        set_value = 'Error: Not handled'
+
+        set_value = setting["Value"]  # Setting value
+        set_type = setting["Type"]  # Setting type        
+
+        # Handle different types of settings
+        if set_type in ['text', 'string', 'password', 'readonly', 'text.select']:
+            value = str(set_value)
+        elif set_type in ['boolean', 'integer.checkbox']:
+            
+            value = False
+
+            if isinstance(set_value, str) and set_value.lower() in ['true', '1']:
+                value = True
+            elif isinstance(set_value, int) and set_value == 1:
+                value = True
+            elif isinstance(set_value, bool):
+                value = set_value
+            
+        elif set_type in ['integer.select', 'integer']:
+            value = int(set_value)
+        elif set_type in ['text.multiselect', 'list', 'subnets']:
+            #  Handle string 
+
+            mylog('debug', [f'[SETTINGS] Handling set_type: "{set_type}", set_value: "{set_value}"'])
+
+            if isinstance(set_value, str):
+                value = json.loads(set_value.replace("'", "\""))
+            # Assuming set_value is a list in this case
+            elif isinstance(set_value, list):
+                value = set_value
+        elif set_type == '.template':
+            # Assuming set_value is a JSON object in this case
+            value = json.loads(set_value)
+        else:
+            mylog('none', [f'[SETTINGS] ⚠ ERROR - set_type not handled:{set_type}'])
+            mylog('none', [f'[SETTINGS] ⚠ ERROR - setting json:{json.dumps(setting)}'])
+
+
+    return value
+
+
+#-------------------------------------------------------------------------------
+# Generate a WHERE condition for SQLite based on a list of values.
+def list_to_where(logical_operator, column_name, condition_operator, values_list):
+    """
+    Generate a WHERE condition for SQLite based on a list of values.
+
+    Parameters:
+    - logical_operator: The logical operator ('AND' or 'OR') to combine conditions.
+    - column_name: The name of the column to filter on.
+    - condition_operator: The condition operator ('LIKE', 'NOT LIKE', '=', '!=', etc.).
+    - values_list: A list of values to be included in the condition.
+
+    Returns:
+    - A string representing the WHERE condition.
+    """
+
+    if not values_list:
+        return ""  # Return an empty string if the list is empty to avoid breaking the SQL condition.
+
+    # Replace {s-quote} with single quote in values_list
+    values_list = [value.replace("{s-quote}", "'") for value in values_list]
+
+    # Build the WHERE condition
+    condition = f"{column_name} {condition_operator} '{values_list[0]}'"
+
+    for value in values_list[1:]:
+        condition += f" {logical_operator} {column_name} {condition_operator} '{value}'"
+
+    return f' AND ({condition}) ' 
+
+
+
+#-------------------------------------------------------------------------------
+# IP validation methods
+#-------------------------------------------------------------------------------
+#-------------------------------------------------------------------------------
+def checkIPV4(ip):
+    """ Define a function to validate an Ip address
+    """
+    ipRegex = "^((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])$"
+
+    if(re.search(ipRegex, ip)):
+        return True
+    else:
+        return False
+
+#-------------------------------------------------------------------------------
+def check_IP_format (pIP):
+    # check if TCP communication error ocurred 
+    if 'communications error to' in pIP:
+        return ''
+
+    # Check IP format 
+    IPv4SEG  = r'(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])'
+    IPv4ADDR = r'(?:(?:' + IPv4SEG + r'\.){3,3}' + IPv4SEG + r')'
+    IP = re.search(IPv4ADDR, pIP)
+
+    # Return empty if not IP
+    if IP is None :
+        return ""
+
+    # Return IP
+    return IP.group(0)
+
+
+
+#-------------------------------------------------------------------------------
+def get_device_name_nslookup(db, pMAC, pIP):
+    
+    nameNotFound = "(name not found)"
+
+    sql = db.sql
+
+    name = nameNotFound
+    
+    #  get names from the NSLOOKUP plugin entries vased on MAC
+    sql.execute(
+        f"""
+         SELECT Watched_Value2 FROM Plugins_Objects 
+         WHERE 
+            Plugin = 'NSLOOKUP' AND 
+            Object_PrimaryID = '{pMAC}'
+         """
+    )         
+    nslookupEntry = sql.fetchall() 
+    db.commitDB()
+
+    if len(nslookupEntry) != 0:
+        name = cleanDeviceName(nslookupEntry[0][0], False)
+
+        return name
+
+    #  get names from the NSLOOKUP plugin entries based on IP
+    sql.execute(
+        f"""
+         SELECT Watched_Value2 FROM Plugins_Objects 
+         WHERE 
+            Plugin = 'NSLOOKUP' AND             
+            Object_SecondaryID = '{pIP}'
+         """
+    )         
+    nslookupEntry = sql.fetchall() 
+    db.commitDB()
+
+    if len(nslookupEntry) != 0:
+        name = cleanDeviceName(nslookupEntry[0][0], True)
+
+        return name
+
+    return name
+
+
+#-------------------------------------------------------------------------------
+def resolve_device_name_dig (pMAC, pIP):
+    
+    nameNotFound = "(name not found)"
+
+    dig_args = ['dig', '+short', '-x', pIP]
+
+    # Execute command
+    try:
+        # try runnning a subprocess
+        newName = subprocess.check_output (dig_args, universal_newlines=True)
+
+        # Check returns
+        newName = newName.strip()
+
+        if len(newName) == 0 :
+            return nameNotFound
+            
+        # Cleanup
+        newName = cleanDeviceName(newName, True)
+
+        if newName == "" or  len(newName) == 0 or newName == '-1' or newName == -1 or "communications error" in newName or 'malformed message packet' in newName : 
+            return nameNotFound
+
+        # all checks passed
+        mylog('debug', [f'[resolve_device_name_dig] Found a new name: "{newName}"'])  
+
+        return newName
+
+    except subprocess.CalledProcessError as e:
+        # An error occured, handle it
+        mylog('none', ['[resolve_device_name_dig] ⚠ ERROR: ', e.output])            
+        # newName = "Error - check logs"
+        return nameNotFound
+
+
+#-------------------------------------------------------------------------------
+# DNS record (Pholus/Name resolution) cleanup methods
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Disclaimer - I'm interfacing with a script I didn't write (pholus3.py) so it's possible I'm missing types of answers
+# it's also possible the pholus3.py script can be adjusted to provide a better output to interface with it
+# Hit me with a PR if you know how! :)
+def resolve_device_name_pholus (pMAC, pIP, allRes, nameNotFound, match_IP = False):
+    
+    pholusMatchesIndexes = []
+
+    result = nameNotFound
+
+    # Collect all Pholus entries  with matching MAC and of type Answer
+    index = 0
+    for result in allRes:
+        #  limiting entries used for name resolution to the ones containing the current IP (v4 only)
+        if ((match_IP and result["IP_v4_or_v6"] == pIP ) or ( result["MAC"] == pMAC )) and result["Record_Type"] == "Answer" and '._googlezone' not in result["Value"]:
+            # found entries with a matching MAC address, let's collect indexes             
+            pholusMatchesIndexes.append(index)
+
+        index += 1
+
+    # return if nothing found
+    if len(pholusMatchesIndexes) == 0:
+        return nameNotFound   
+
+    # we have some entries let's try to select the most useful one
+    # Do I need to pre-order allRes to have the most valuable onse on the top?
+
+    for i in pholusMatchesIndexes:
+        if not checkIPV4(allRes[i]['IP_v4_or_v6']):
+            continue
+
+        value = allRes[i]["Value"]
+
+        # airplay matches contain a lot of information
+        # Matches for example:
+        # Brand Tv (50)._airplay._tcp.local. TXT Class:32769 "acl=0 deviceid=66:66:66:66:66:66 features=0x77777,0x38BCB46 rsf=0x3 fv=p20.T-FFFFFF-03.1 flags=0x204 model=XXXX manufacturer=Brand serialNumber=XXXXXXXXXXX protovers=1.1 srcvers=777.77.77 pi=FF:FF:FF:FF:FF:FF psi=00000000-0000-0000-0000-FFFFFFFFFF gid=00000000-0000-0000-0000-FFFFFFFFFF gcgl=0 pk=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+        if '._airplay._tcp.local. TXT Class:32769' in value:
+            return cleanDeviceName(value.split('._airplay._tcp.local. TXT Class:32769')[0], match_IP)
+        
+        # second best - contains airplay
+        # Matches for example:
+        # _airplay._tcp.local. PTR Class:IN "Brand Tv (50)._airplay._tcp.local."
+        if '_airplay._tcp.local. PTR Class:IN' in value and ('._googlecast') not in value:
+            return cleanDeviceName(value.split('"')[1], match_IP)    
+
+        # Contains PTR Class:32769
+        # Matches for example:
+        # 3.1.168.192.in-addr.arpa. PTR Class:32769 "MyPc.local."
+        if 'PTR Class:32769' in value:
+            return cleanDeviceName(value.split('"')[1], match_IP)
+
+        # Contains AAAA Class:IN
+        # Matches for example:
+        # DESKTOP-SOMEID.local. AAAA Class:IN "fe80::fe80:fe80:fe80:fe80"
+        if 'AAAA Class:IN' in value:
+            return cleanDeviceName(value.split('.local.')[0], match_IP)
+
+        # Contains _googlecast._tcp.local. PTR Class:IN
+        # Matches for example:
+        # _googlecast._tcp.local. PTR Class:IN "Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77._googlecast._tcp.local."
+        if '_googlecast._tcp.local. PTR Class:IN' in value and ('Google-Cast-Group') not in value:
+            return cleanDeviceName(value.split('"')[1], match_IP)
+
+        # Contains A Class:32769
+        # Matches for example:
+        # Android.local. A Class:32769 "192.168.1.6"
+        if ' A Class:32769' in value:
+            return cleanDeviceName(value.split(' A Class:32769')[0], match_IP)
+
+        # Contains PTR Class:IN
+        # Matches for example:
+        # _esphomelib._tcp.local. PTR Class:IN "ceiling-light-1._esphomelib._tcp.local."
+        if 'PTR Class:IN' in value and len(value.split('"')) > 1:
+            return cleanDeviceName(value.split('"')[1], match_IP)
+
+    return nameNotFound
+    
+    
+
+#-------------------------------------------------------------------------------
+def cleanDeviceName(str, match_IP):
+    # alternative str.split('.')[0]
+    str = str.replace("._airplay", "")
+    str = str.replace("._tcp", "")
+    str = str.replace(".localdomain", "")
+    str = str.replace(".local", "")    
+    str = str.replace("._esphomelib", "")
+    str = str.replace("._googlecast", "")
+    str = str.replace(".lan", "")
+    str = str.replace(".home", "")
+    str = re.sub(r'-[a-fA-F0-9]{32}', '', str)    # removing last part of e.g. Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77
+    str = re.sub(r'#.*', '', str) # Remove everything after '#' including the '#'
+    # remove trailing dots
+    if str.endswith('.'):
+        str = str[:-1]
+
+
+    if match_IP:
+        str = str + " (IP match)"
+
+    return str
+
+
+#-------------------------------------------------------------------------------
+# String manipulation methods
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+
+def bytes_to_string(value):
+    # if value is of type bytes, convert to string
+    if isinstance(value, bytes):
+        value = value.decode('utf-8')
+    return value
+
+#-------------------------------------------------------------------------------
+
+def if_byte_then_to_str(input):
+    if isinstance(input, bytes):
+        input = input.decode('utf-8')
+        input = bytes_to_string(re.sub('[^a-zA-Z0-9-_\s]', '', str(input)))
+    return input
+
+#-------------------------------------------------------------------------------
+def hide_email(email):
+    m = email.split('@')
+
+    if len(m) == 2:
+        return f'{m[0][0]}{"*"*(len(m[0])-2)}{m[0][-1] if len(m[0]) > 1 else ""}@{m[1]}'
+
+    return email
+
+#-------------------------------------------------------------------------------
+def hide_string(input_string):
+    if len(input_string) < 3:
+        return input_string  # Strings with 2 or fewer characters remain unchanged
+    else:
+        return input_string[0] + "*" * (len(input_string) - 2) + input_string[-1]
+
+
+#-------------------------------------------------------------------------------
+def removeDuplicateNewLines(text):
+    if "\n\n\n" in text:
+        return removeDuplicateNewLines(text.replace("\n\n\n", "\n\n"))
+    else:
+        return text
+
+#-------------------------------------------------------------------------------
+
+def sanitize_string(input):
+    if isinstance(input, bytes):
+        input = input.decode('utf-8')
+    value = bytes_to_string(re.sub('[^a-zA-Z0-9-_\s]', '', str(input)))
+    return value
+
+
+#-------------------------------------------------------------------------------
+def generate_mac_links (html, deviceUrl):
+
+    p = re.compile(r'(?:[0-9a-fA-F]:?){12}')
+
+    MACs = re.findall(p, html)
+
+    for mac in MACs:
+        html = html.replace('<td>' + mac + '</td>','<td><a href="' + deviceUrl + mac + '">' + mac + '</a></td>')
+
+    return html
+
+#-------------------------------------------------------------------------------
+# JSON methods
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+def isJsonObject(value):
+    return isinstance(value, dict)
+
+#-------------------------------------------------------------------------------
+def add_json_list (row, list):
+    new_row = []
+    for column in row :
+        column = bytes_to_string(column)
+
+        new_row.append(column)
+
+    list.append(new_row)
+
+    return list
+
+
+#-------------------------------------------------------------------------------
+# Checks if the object has a __dict__ attribute. If it does, it assumes that it's an instance of a class and serializes its attributes dynamically. 
+class AppStateEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if hasattr(obj, '__dict__'):
+            # If the object has a '__dict__', assume it's an instance of a class
+            return obj.__dict__
+        return super().default(obj)
+
+#-------------------------------------------------------------------------------
+# Checks if the object has a __dict__ attribute. If it does, it assumes that it's an instance of a class and serializes its attributes dynamically. 
+class NotiStrucEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if hasattr(obj, '__dict__'):
+            # If the object has a '__dict__', assume it's an instance of a class
+            return obj.__dict__
+        return super().default(obj)
+
+#-------------------------------------------------------------------------------
+#  Creates a JSON object from a DB row
+def row_to_json(names, row):
+
+    rowEntry = {}
+
+    index = 0
+    for name in names:
+        rowEntry[name]= if_byte_then_to_str(row[name])
+        index += 1
+
+    return rowEntry
+
+#-------------------------------------------------------------------------------
+# Get language strings from plugin JSON
+def collect_lang_strings(json, pref, stringSqlParams):    
+
+    for prop in json["localized"]:
+        for language_string in json[prop]:
+
+            stringSqlParams.append((str(language_string["language_code"]), str(pref + "_" + prop), str(language_string["string"]), ""))
+
+
+    return stringSqlParams
+
+
+#-------------------------------------------------------------------------------
+#  Misc
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+def checkNewVersion():
+
+    mylog('debug', [f"[Version check] Checking if new version available"])
+
+    newVersion = False
+
+    f = open(applicationPath + '/front/buildtimestamp.txt', 'r')
+    buildTimestamp = int(f.read().strip())
+    f.close()
+
+    data = ""
+
+    try:
+        url = requests.get("https://api.github.com/repos/jokob-sk/NetAlertX/releases")
+        text = url.text
+        data = json.loads(text)
+    except requests.exceptions.ConnectionError as e:
+        mylog('minimal', ["[Version check] ⚠ ERROR: Couldn't check for new release."])
+        data = ""
+
+    # make sure we received a valid response and not an API rate limit exceeded message
+    if data != "" and len(data) > 0 and isinstance(data, list) and "published_at" in data[0]:
+
+        dateTimeStr = data[0]["published_at"]
+
+        releaseTimestamp = int(datetime.datetime.strptime(dateTimeStr, '%Y-%m-%dT%H:%M:%S%z').timestamp())
+
+        if releaseTimestamp > buildTimestamp + 600:
+            mylog('none', ["[Version check] New version of the container available!"])
+            newVersion = True       
+        else:
+            mylog('none', ["[Version check] Running the latest version."])
+
+    return newVersion
+
+
+#-------------------------------------------------------------------------------
+def initOrSetParam(db, parID, parValue):
+    sql = db.sql
+
+    sql.execute ("INSERT INTO Parameters(par_ID, par_Value) VALUES('"+str(parID)+"', '"+str(parValue)+"') ON CONFLICT(par_ID) DO UPDATE SET par_Value='"+str(parValue)+"' where par_ID='"+str(parID)+"'")
+
+    db.commitDB()
+
+#-------------------------------------------------------------------------------
+class json_obj:
+    def __init__(self, jsn, columnNames):
+        self.json = jsn
+        self.columnNames = columnNames
+
+#-------------------------------------------------------------------------------
+class noti_obj:
+    def __init__(self, json, text, html):
+        self.json = json
+        self.text = text
+        self.html = html  
+

server/initialise.py

@@ -0,0 +1,314 @@
+
+import os
+import time
+from pytz import timezone
+from cron_converter import Cron
+from pathlib import Path
+import datetime
+import json
+import shutil
+import re
+
+
+import conf 
+from const import fullConfPath
+from helper import collect_lang_strings, updateSubnets, initOrSetParam, isJsonObject, updateState
+from logger import mylog
+from api import update_api
+from scheduler import schedule_class
+from plugin import print_plugin_info, run_plugin_scripts
+from plugin_utils import get_plugins_configs
+
+#===============================================================================
+# Initialise user defined values
+#===============================================================================
+
+#-------------------------------------------------------------------------------
+# Import user values
+# Check config dictionary
+def ccd(key, default, config_dir, name, inputtype, options, group, events=[], desc = "", regex = "", setJsonMetadata = {}, overrideTemplate = {}):
+
+    # use default inintialization value    
+    result = default
+
+    if events is None:
+        events = []
+
+    # use existing value if already supplied, otherwise default value is used
+    if key in config_dir:
+        result =  config_dir[key]
+
+    if inputtype == 'text':
+        result = result.replace('\'', "{s-quote}")
+
+    conf.mySettingsSQLsafe.append((key, name, desc, inputtype, options, regex, str(result), group, str(events)))
+    conf.mySettings.append((key, name, desc, inputtype, options, regex, result, group, str(events)))
+
+    # save metadata in dummy setting
+    if '__metadata' not in key:
+        tuple = (f'{key}__metadata', "metadata name", "metadata desc", 'json', "", "", json.dumps(setJsonMetadata), group, '[]')
+        conf.mySettingsSQLsafe.append(tuple)
+        conf.mySettings.append(tuple)
+    
+        
+
+    return result
+#-------------------------------------------------------------------------------
+
+def importConfigs (db): 
+
+    sql = db.sql
+
+    # get config file name
+    config_file = Path(fullConfPath)
+
+    # Only import file if the file was modifed since last import.
+    # this avoids time zone issues as we just compare the previous timestamp to the current time stamp
+
+    # rename settings that have changed names due to code cleanup and migration to plugins
+    renameSettings(config_file)
+
+    fileModifiedTime = os.path.getmtime(config_file)
+
+    mylog('debug', ['[Import Config] checking config file '])
+    mylog('debug', ['[Import Config] lastImportedConfFile     :', conf.lastImportedConfFile])
+    mylog('debug', ['[Import Config] fileModifiedTime         :', fileModifiedTime])
+    
+
+    if (fileModifiedTime == conf.lastImportedConfFile) :
+        mylog('debug', ['[Import Config] skipping config file import'])
+        return
+
+    # Header
+    updateState("Import config", showSpinner = True)  
+
+    # remove all plugin langauge strings
+    sql.execute("DELETE FROM Plugins_Language_Strings;")
+    db.commitDB()
+    
+    mylog('debug', ['[Import Config] importing config file'])
+    conf.mySettings = [] # reset settings
+    conf.mySettingsSQLsafe = [] # same as above but safe to be passed into a SQL query
+
+    # User values loaded from now
+    c_d = read_config_file(config_file)
+
+   
+    # Import setting if found in the dictionary
+    
+    # General    
+    # ----------------------------------------
+    
+    conf.LOG_LEVEL = ccd('LOG_LEVEL', 'verbose' , c_d, 'Log verboseness', 'text.select', "['none', 'minimal', 'verbose', 'debug']", 'General')
+    conf.TIMEZONE = ccd('TIMEZONE', 'Europe/Berlin' , c_d, 'Time zone', 'text', '', 'General')    
+    conf.PLUGINS_KEEP_HIST = ccd('PLUGINS_KEEP_HIST', 250 , c_d, 'Keep history entries', 'integer', '', 'General') 
+    conf.PIALERT_WEB_PROTECTION = ccd('PIALERT_WEB_PROTECTION', False , c_d, 'Enable logon', 'boolean', '', 'General')
+    conf.PIALERT_WEB_PASSWORD = ccd('PIALERT_WEB_PASSWORD', '8d969eef6ecad3c29a3a629280e686cf0c3f5d5a86aff3ca12020c923adc6c92' , c_d, 'Logon password', 'readonly', '', 'General')    
+    conf.REPORT_DASHBOARD_URL = ccd('REPORT_DASHBOARD_URL', 'http://netalertx/' , c_d, 'NetAlertX URL', 'text', '', 'General')
+    conf.UI_LANG = ccd('UI_LANG', 'English' , c_d, 'Language Interface', 'text.select', "['English', 'French', 'German', 'Norwegian', 'Russian', 'Spanish' ]", 'General')
+    conf.UI_PRESENCE = ccd('UI_PRESENCE', ['online', 'offline', 'archived']   , c_d, 'Include in presence', 'text.multiselect', "['online', 'offline', 'archived']", 'General')    
+    conf.UI_DEV_SECTIONS = ccd('UI_DEV_SECTIONS', []   , c_d, 'Show sections', 'text.multiselect', "['Tile Cards', 'Device Presence']", 'General')    
+    conf.UI_MY_DEVICES = ccd('UI_MY_DEVICES', ['online', 'offline', 'archived', 'new', 'down']   , c_d, 'Include in My Devices', 'text.multiselect', "['online', 'offline', 'archived', 'new', 'down']", 'General')    
+    conf.UI_NOT_RANDOM_MAC = ccd('UI_NOT_RANDOM_MAC', []   , c_d, 'Exlude from Random Prefix', 'list', "", 'General')    
+    conf.UI_ICONS = ccd('UI_ICONS', ['PGkgY2xhc3M9ImZhIGZhLWNvbXB1dGVyIj48L2k', 'PGkgY2xhc3M9ImZhIGZhLWV0aGVybmV0Ij48L2k', 'PGkgY2xhc3M9ImZhIGZhLWdhbWVwYWQiPjwvaT4', 'PGkgY2xhc3M9ImZhIGZhLWdsb2JlIj48L2k', 'PGkgY2xhc3M9ImZhIGZhLWxhcHRvcCI', 'PGkgY2xhc3M9ImZhIGZhLWxpZ2h0YnVsYiI', 'PGkgY2xhc3M9ImZhIGZhLXNoaWVsZCI', 'PGkgY2xhc3M9ImZhIGZhLXdpZmkiPjwvaT4']  , c_d, 'Icons', 'list', "", 'General')    
+    conf.UI_REFRESH = ccd('UI_REFRESH', 0 , c_d, 'Refresh interval', 'integer', "", 'General')    
+    conf.DAYS_TO_KEEP_EVENTS = ccd('DAYS_TO_KEEP_EVENTS', 90 , c_d, 'Delete events days', 'integer', '', 'General')
+    conf.HRS_TO_KEEP_NEWDEV = ccd('HRS_TO_KEEP_NEWDEV', 0 , c_d, 'Keep new devices for', 'integer', "0", 'General')        
+    conf.API_CUSTOM_SQL = ccd('API_CUSTOM_SQL', 'SELECT * FROM Devices WHERE dev_PresentLastScan = 0' , c_d, 'Custom endpoint', 'text', '', 'General')
+    conf.NETWORK_DEVICE_TYPES = ccd('NETWORK_DEVICE_TYPES', ['AP', 'Gateway', 'Firewall', 'Hypervisor', 'Powerline', 'Switch', 'WLAN', 'PLC', 'Router','USB LAN Adapter', 'USB WIFI Adapter', 'Internet'] , c_d, 'Network device types', 'list', '', 'General')
+
+    # ARPSCAN (+ more settings are provided by the ARPSCAN plugin)    
+    conf.SCAN_SUBNETS = ccd('SCAN_SUBNETS', ['192.168.1.0/24 --interface=eth1', '192.168.1.0/24 --interface=eth0'] , c_d, 'Subnets to scan', 'subnets', '', 'ARPSCAN')    
+
+    #  Init timezone in case it changed
+    conf.tz = timezone(conf.TIMEZONE) 
+
+    # TODO cleanup later ----------------------------------------------------------------------------------
+    # init all time values as we have timezone - all this shoudl be moved into plugin/plugin settings
+    conf.time_started = datetime.datetime.now(conf.tz)    
+    conf.plugins_once_run = False
+
+    # timestamps of last execution times
+    conf.startTime  = conf.time_started
+    now_minus_24h   = conf.time_started - datetime.timedelta(hours = 24)
+
+    # set these times to the past to force the first run         
+    conf.last_scan_run          = now_minus_24h        
+    conf.last_version_check     = now_minus_24h  
+
+    # TODO cleanup later ----------------------------------------------------------------------------------
+    
+    # reset schedules
+    conf.mySchedules = [] 
+
+    # Format and prepare the list of subnets
+    conf.userSubnets = updateSubnets(conf.SCAN_SUBNETS)
+
+    # Plugins START
+    # -----------------
+    conf.plugins = get_plugins_configs()
+
+    mylog('none', ['[Config] Plugins: Number of dynamically loaded plugins: ', len(conf.plugins)])
+
+    #  handle plugins
+    index = 0
+    for plugin in conf.plugins:
+        # Header
+        updateState(f"Import plugin {index} of {len(conf.plugins)}") 
+        index +=1
+
+        pref = plugin["unique_prefix"]  
+        print_plugin_info(plugin, ['display_name','description'])
+
+        # if plugin["enabled"] == 'true': 
+
+        stringSqlParams = []
+        
+        # collect plugin level language strings
+        stringSqlParams = collect_lang_strings(plugin, pref, stringSqlParams)
+        
+        for set in plugin["settings"]:
+            setFunction = set["function"]
+            # Setting code name / key  
+            key = pref + "_" + setFunction 
+
+            # set.get() - returns None if not found, set["options"] raises error
+            #  ccd(key, default, config_dir, name, inputtype, options, group, events=[], desc = "", regex = "", setJsonMetadata = {}):
+            v = ccd(key, 
+                    set["default_value"], 
+                    c_d, 
+                    set["name"][0]["string"], 
+                    set["type"] , 
+                    str(set["options"]), 
+                    group = pref, 
+                    events = set.get("events"), 
+                    desc = set["description"][0]["string"], 
+                    regex = "", 
+                    setJsonMetadata = set)                   
+
+            # Save the user defined value into the object
+            set["value"] = v
+
+            # Setup schedules
+            if setFunction == 'RUN_SCHD':
+                newSchedule = Cron(v).schedule(start_date=datetime.datetime.now(conf.tz))
+                conf.mySchedules.append(schedule_class(pref, newSchedule, newSchedule.next(), False))
+
+            # Collect settings related language strings
+            # Creates an entry with key, for example ARPSCAN_CMD_name
+            stringSqlParams = collect_lang_strings(set,  pref + "_" + set["function"], stringSqlParams)
+
+        # Collect column related language strings
+        for clmn in plugin.get('database_column_definitions', []):
+            # Creates an entry with key, for example ARPSCAN_Object_PrimaryID_name
+            stringSqlParams = collect_lang_strings(clmn,  pref + "_" + clmn.get("column", ""), stringSqlParams)
+
+        #  bulk-import language strings
+        sql.executemany ("""INSERT INTO Plugins_Language_Strings ("Language_Code", "String_Key", "String_Value", "Extra") VALUES (?, ?, ?, ?)""", stringSqlParams )
+
+        # db.commitDB()
+
+
+
+    conf.plugins_once_run = False
+    # -----------------
+    # Plugins END
+  
+
+    # Insert settings into the DB    
+    sql.execute ("DELETE FROM Settings")    
+    sql.executemany ("""INSERT INTO Settings ("Code_Name", "Display_Name", "Description", "Type", "Options",
+         "RegEx", "Value", "Group", "Events" ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", conf.mySettingsSQLsafe)
+    
+    db.commitDB()
+
+    #  update only the settings datasource
+    update_api(db, False, ["settings"])  
+    
+    # run plugins that are modifying the config   
+    run_plugin_scripts(db, 'before_config_save' )
+
+    # Used to determine the next import
+    conf.lastImportedConfFile = os.path.getmtime(config_file)   
+
+    updateState("Config imported", conf.lastImportedConfFile, conf.lastImportedConfFile, False)   
+
+    mylog('minimal', '[Config] Imported new config')
+
+
+
+#-------------------------------------------------------------------------------
+def read_config_file(filename):
+    """
+    retuns dict on the config file key:value pairs
+    """
+    mylog('minimal', '[Config] reading config file')
+    # load the variables from .conf file
+    code = compile(filename.read_text(), filename.name, "exec")
+    confDict = {} # config dictionary
+    exec(code, {"__builtins__": {}}, confDict)
+    return confDict 
+
+
+#-------------------------------------------------------------------------------
+# DEPERECATED soonest after 10/10/2024
+# 🤔Idea/TODO: Check and compare versions/timestamps amd only perform a replacement if config/version older than...
+replacements = {
+    r'\bREPORT_TO\b': 'SMTP_REPORT_TO',
+    r'\bREPORT_FROM\b': 'SMTP_REPORT_FROM',
+    r'REPORT_MAIL=True': 'SMTP_RUN=\'on_notification\'',
+    r'REPORT_APPRISE=True': 'APPRISE_RUN=\'on_notification\'',
+    r'REPORT_NTFY=True': 'NTFY_RUN=\'on_notification\'',
+    r'REPORT_WEBHOOK=True': 'WEBHOOK_RUN=\'on_notification\'',
+    r'REPORT_PUSHSAFER=True': 'PUSHSAFER_RUN=\'on_notification\'',
+    r'REPORT_MQTT=True': 'MQTT_RUN=\'on_notification\'',
+    r'PIHOLE_CMD=': 'PIHOLE_CMD_OLD=',
+    r'\bINCLUDED_SECTIONS\b': 'NTFPRCS_INCLUDED_SECTIONS',
+    r'\bDIG_GET_IP_ARG\b': 'INTRNT_DIG_GET_IP_ARG',
+    r'\/home/pi/pialert\b': '/app'
+}
+
+def renameSettings(config_file):
+    # Check if the file contains any of the old setting code names
+    contains_old_settings = False
+
+    # Open the original config_file for reading
+    with open(str(config_file), 'r') as original_file:  # Convert config_file to a string
+        for line in original_file:
+            # Use regular expressions with word boundaries to check for the old setting code names
+            if any(re.search(key, line) for key in replacements.keys()):
+                contains_old_settings = True
+                break  # Exit the loop if any old setting is found
+
+    # If the file contains old settings, proceed with renaming and backup
+    if contains_old_settings:
+        # Create a backup file with the suffix "_old_setting_names" and timestamp
+        timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
+        backup_file = f"{config_file}_old_setting_names_{timestamp}.bak"
+
+        mylog('debug', f'[Config] Old setting names will be replaced and a backup ({backup_file}) of the config created.')
+
+        shutil.copy(str(config_file), backup_file)  # Convert config_file to a string
+
+        # Open the original config_file for reading and create a temporary file for writing
+        with open(str(config_file), 'r') as original_file, open(str(config_file) + "_temp", 'w') as temp_file:  # Convert config_file to a string
+            for line in original_file:
+                # Use regular expressions with word boundaries for replacements
+                for key, value in replacements.items():
+                    line = re.sub(key, value, line)
+
+                # Write the modified line to the temporary file
+                temp_file.write(line)
+
+        # Close both files
+        original_file.close()
+        temp_file.close()
+
+        # Replace the original config_file with the temporary file
+        shutil.move(str(config_file) + "_temp", str(config_file))  # Convert config_file to a string
+    else:
+        mylog('debug', '[Config] No old setting names found in the file. No changes made.')
+
+        
+
+ 

server/logger.py

@@ -0,0 +1,142 @@
+""" Colection of functions to support all logging for NetAlertX """
+import sys
+import io
+import datetime
+import threading
+import time
+
+import conf
+from const import *
+# from helper import get_setting_value
+
+#-------------------------------------------------------------------------------
+# duplication from helper to avoid circle
+#-------------------------------------------------------------------------------
+def timeNowTZ():
+    if conf.tz:
+        return datetime.datetime.now(conf.tz).replace(microsecond=0)
+    else:
+        return datetime.datetime.now().replace(microsecond=0)
+
+
+#-------------------------------------------------------------------------------
+# More verbose as the numbers go up
+debugLevels =   [
+                    ('none', 0), ('minimal', 1), ('verbose', 2), ('debug', 3)
+                ]
+
+currentLevel = 0
+
+def mylog(requestedDebugLevel, n):
+
+    setLvl = 0  
+    reqLvl = 0  
+
+    #  Get debug urgency/relative weight
+    for lvl in debugLevels:
+        if conf.LOG_LEVEL == lvl[0]:
+            setLvl = lvl[1]
+        if requestedDebugLevel == lvl[0]:
+            reqLvl = lvl[1]
+
+    if reqLvl <= setLvl:
+        file_print (*n)        
+
+#-------------------------------------------------------------------------------
+def file_print (*args):
+
+    result = timeNowTZ().strftime ('%H:%M:%S') + ' '    
+       
+    for arg in args:                
+        result += str(arg)        
+    print(result)
+ 
+    append_to_file_with_timeout(logPath + "/app.log", result + '\n', 5)
+
+#-------------------------------------------------------------------------------
+# Function to append to the file
+def append_to_file(file_path, data):
+    try:
+        # Open the file for appending
+        file = open(file_path, "a")
+        
+        # Write the data to the file
+        file.write(data)
+        
+        # Close the file
+        file.close()
+    except Exception as e:
+        print(f"Error appending to file: {e}")
+
+#-------------------------------------------------------------------------------
+# Function to append to the file with a timeout
+def append_to_file_with_timeout(file_path, data, timeout):
+    # Create a thread for appending to the file
+    append_thread = threading.Thread(target=append_to_file, args=(file_path, data))
+    
+    # Start the thread
+    append_thread.start()
+    
+    # Wait for the thread to complete or timeout
+    append_thread.join(timeout)
+    
+    # If the thread is still running, it has exceeded the timeout
+    if append_thread.is_alive():
+        append_thread.join()  # Optionally, you can force it to terminate
+        
+        # Handle the timeout here, e.g., log an error
+        print("Appending to file timed out")
+
+
+
+#-------------------------------------------------------------------------------
+def print_log (pText):
+
+    # Check LOG actived
+    if not conf.LOG_LEVEL == 'debug' :
+        return
+
+    # Current Time    
+    log_timestamp2 = datetime.datetime.now(conf.tz).replace(microsecond=0)
+
+    # Print line + time + elapsed time + text
+    file_print ('[LOG_LEVEL=debug] ',
+        # log_timestamp2, ' ',
+        log_timestamp2.strftime ('%H:%M:%S'), ' ',
+        pText)
+
+
+    return pText
+
+
+
+#-------------------------------------------------------------------------------
+# textchars = bytearray({7,8,9,10,12,13,27} | set(range(0x20, 0x100)) - {0x7f})
+# is_binary_string = lambda bytes: bool(bytes.translate(None, textchars))
+
+def append_file_binary(file_path, input_data):
+    with open(file_path, 'ab') as file:
+        if isinstance(input_data, str):
+            input_data = input_data.encode('utf-8')  # Encode string as bytes
+        file.write(input_data)
+
+
+
+#-------------------------------------------------------------------------------
+def logResult (stdout, stderr):
+    if stderr != None:
+        append_file_binary (logPath + '/stderr.log', stderr)
+    if stdout != None:
+        append_file_binary (logPath + '/stdout.log', stdout)  
+
+#-------------------------------------------------------------------------------
+def append_line_to_file (pPath, pText):
+    # append the line depending using the correct python version
+    if sys.version_info < (3, 0):
+        file = io.open (pPath , mode='a', encoding='utf-8')
+        file.write ( pText.decode('unicode_escape') ) 
+        file.close() 
+    else:
+        file = open (pPath, 'a', encoding='utf-8') 
+        file.write (pText) 
+        file.close() 

server/networkscan.py

@@ -0,0 +1,226 @@
+
+
+import conf
+
+from database import insertOnlineHistory
+from device import create_new_devices, print_scan_stats, save_scanned_devices, update_devices_data_from_scan
+from helper import timeNowTZ
+from logger import mylog
+from reporting import skip_repeated_notifications
+
+
+
+#===============================================================================
+# SCAN NETWORK
+#===============================================================================
+
+def process_scan (db):
+
+     # Load current scan data
+    mylog('verbose','[Process Scan]  Processing scan results')     
+    save_scanned_devices (db)    
+
+    db.commitDB()
+    
+    # Print stats
+    mylog('none','[Process Scan] Print Stats')
+    print_scan_stats(db)
+    mylog('none','[Process Scan] Stats end')
+
+    # Create Events    
+    mylog('verbose','[Process Scan] Sessions Events (connect / discconnect)')
+    insert_events(db)
+
+    # Create New Devices
+    # after create events -> avoid 'connection' event
+    mylog('verbose','[Process Scan] Creating new devices')
+    create_new_devices (db)
+
+    # Update devices info
+    mylog('verbose','[Process Scan] Updating Devices Info')
+    update_devices_data_from_scan (db)
+
+    # Void false connection - disconnections
+    mylog('verbose','[Process Scan] Voiding false (ghost) disconnections')    
+    void_ghost_disconnections (db)
+
+    # Pair session events (Connection / Disconnection)
+    mylog('verbose','[Process Scan] Pairing session events (connection / disconnection) ')
+    pair_sessions_events(db)  
+  
+    # Sessions snapshot
+    mylog('verbose','[Process Scan] Creating sessions snapshot')
+    create_sessions_snapshot (db)
+
+    # Sessions snapshot
+    mylog('verbose','[Process Scan] Inserting scan results into Online_History')
+    insertOnlineHistory(db)
+  
+    # Skip repeated notifications
+    mylog('verbose','[Process Scan] Skipping repeated notifications')
+    skip_repeated_notifications (db)
+
+    # Clear current scan as processed 
+    db.sql.execute ("DELETE FROM CurrentScan") 
+  
+    # Commit changes    
+    db.commitDB()
+
+#-------------------------------------------------------------------------------
+def void_ghost_disconnections (db):
+    sql = db.sql #TO-DO
+    startTime = timeNowTZ()
+    # Void connect ghost events (disconnect event exists in last X min.) 
+    mylog('debug','[Void Ghost Con] - 1 Connect ghost events')
+    sql.execute("""UPDATE Events SET eve_PairEventRowid = Null,
+                                eve_EventType ='VOIDED - ' || eve_EventType
+                            WHERE eve_MAC != 'Internet'
+                            AND eve_EventType = 'Connected'
+                            AND eve_DateTime = ?
+                            AND eve_MAC IN (
+                                SELECT Events.eve_MAC
+                                FROM CurrentScan, Devices, Events 
+                                WHERE dev_MAC = cur_MAC
+                                    AND eve_MAC = cur_MAC
+                                    AND eve_EventType = 'Disconnected'
+                                    AND eve_DateTime >= DATETIME(?, '-3 minutes')
+                                ) """,
+                            (startTime, startTime))
+
+    # Void connect paired events
+    mylog('debug','[Void Ghost Con] - 2 Paired events')
+    sql.execute("""UPDATE Events SET eve_PairEventRowid = Null 
+                            WHERE eve_MAC != 'Internet'
+                            AND eve_PairEventRowid IN (
+                                SELECT Events.RowID
+                                FROM CurrentScan, Devices, Events 
+                                WHERE dev_MAC = cur_MAC
+                                    AND eve_MAC = cur_MAC
+                                    AND eve_EventType = 'Disconnected'
+                                    AND eve_DateTime >= DATETIME(?, '-3 minutes')
+                                ) """,
+                            (startTime,))
+
+    # Void disconnect ghost events 
+    mylog('debug','[Void Ghost Con] - 3 Disconnect ghost events')
+    sql.execute("""UPDATE Events SET eve_PairEventRowid = Null, 
+                            eve_EventType = 'VOIDED - '|| eve_EventType
+                        WHERE eve_MAC != 'Internet'
+                        AND ROWID IN (
+                            SELECT Events.RowID
+                            FROM CurrentScan, Devices, Events 
+                            WHERE dev_MAC = cur_MAC
+                                AND eve_MAC = cur_MAC
+                                AND eve_EventType = 'Disconnected'
+                                AND eve_DateTime >= DATETIME(?, '-3 minutes')
+                            ) """,
+                (startTime,))
+
+    mylog('debug','[Void Ghost Con] Void Ghost Connections end')
+
+    db.commitDB()
+
+#-------------------------------------------------------------------------------
+def pair_sessions_events (db):
+    sql = db.sql #TO-DO
+   
+
+    # Pair Connection / New Device events
+    mylog('debug','[Pair Session] - 1 Connections / New Devices')
+    sql.execute ("""UPDATE Events
+                    SET eve_PairEventRowid =
+                       (SELECT ROWID
+                        FROM Events AS EVE2
+                        WHERE EVE2.eve_EventType IN ('New Device', 'Connected',
+                            'Device Down', 'Disconnected')
+                           AND EVE2.eve_MAC = Events.eve_MAC
+                           AND EVE2.eve_Datetime > Events.eve_DateTime
+                        ORDER BY EVE2.eve_DateTime ASC LIMIT 1)
+                    WHERE eve_EventType IN ('New Device', 'Connected')
+                    AND eve_PairEventRowid IS NULL
+                 """ )
+
+    # Pair Disconnection / Device Down
+    mylog('debug','[Pair Session] - 2 Disconnections')
+    sql.execute ("""UPDATE Events
+                    SET eve_PairEventRowid =
+                        (SELECT ROWID
+                         FROM Events AS EVE2
+                         WHERE EVE2.eve_PairEventRowid = Events.ROWID)
+                    WHERE eve_EventType IN ('Device Down', 'Disconnected')
+                      AND eve_PairEventRowid IS NULL
+                 """ )
+    mylog('debug','[Pair Session] Pair session end')
+
+    db.commitDB()
+
+#-------------------------------------------------------------------------------
+def create_sessions_snapshot (db):
+    sql = db.sql #TO-DO
+
+    # Clean sessions snapshot
+    mylog('debug','[Sessions Snapshot] - 1 Clean')
+    sql.execute ("DELETE FROM SESSIONS" )
+
+    # Insert sessions
+    mylog('debug','[Sessions Snapshot] - 2 Insert')
+    sql.execute ("""INSERT INTO Sessions
+                    SELECT * FROM Convert_Events_to_Sessions""" )
+
+    mylog('debug','[Sessions Snapshot] Sessions end')
+    db.commitDB()
+
+
+#-------------------------------------------------------------------------------
+def insert_events (db):
+    sql = db.sql #TO-DO
+    startTime = timeNowTZ()    
+    
+    # Check device down
+    mylog('debug','[Events] - 1 - Devices down')
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+                        eve_EventType, eve_AdditionalInfo,
+                        eve_PendingAlertEmail)
+                    SELECT dev_MAC, dev_LastIP, '{startTime}', 'Device Down', '', 1
+                    FROM Devices 
+                    WHERE dev_AlertDeviceDown != 0
+                      AND dev_PresentLastScan = 1                      
+                      AND NOT EXISTS (SELECT 1 FROM CurrentScan
+                                      WHERE dev_MAC = cur_MAC
+                                         ) """)
+
+    # Check new connections
+    mylog('debug','[Events] - 2 - New Connections')
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+                        eve_EventType, eve_AdditionalInfo,
+                        eve_PendingAlertEmail)
+                    SELECT cur_MAC, cur_IP, '{startTime}', 'Connected', '', dev_AlertEvents
+                    FROM Devices, CurrentScan
+                    WHERE dev_MAC = cur_MAC  
+                      AND dev_PresentLastScan = 0 """)
+
+    # Check disconnections
+    mylog('debug','[Events] - 3 - Disconnections')
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+                        eve_EventType, eve_AdditionalInfo,
+                        eve_PendingAlertEmail)
+                    SELECT dev_MAC, dev_LastIP, '{startTime}', 'Disconnected', '',
+                        dev_AlertEvents
+                    FROM Devices
+                    WHERE dev_AlertDeviceDown = 0
+                      AND dev_PresentLastScan = 1
+                      AND NOT EXISTS (SELECT 1 FROM CurrentScan
+                                      WHERE dev_MAC = cur_MAC
+                                         ) """)
+
+    # Check IP Changed
+    mylog('debug','[Events] - 4 - IP Changes')
+    sql.execute (f"""INSERT INTO Events (eve_MAC, eve_IP, eve_DateTime,
+                        eve_EventType, eve_AdditionalInfo,
+                        eve_PendingAlertEmail)
+                    SELECT cur_MAC, cur_IP, '{startTime}', 'IP Changed',
+                        'Previous IP: '|| dev_LastIP, dev_AlertEvents
+                    FROM Devices, CurrentScan
+                    WHERE dev_MAC = cur_MAC                        
+                      AND dev_LastIP <> cur_IP """ )
+    mylog('debug','[Events] - Events end')

server/notification.py

@@ -0,0 +1,323 @@
+import datetime
+import json
+import uuid
+import socket
+import subprocess
+from json2table import convert
+
+# Register NetAlertX modules 
+import conf
+from const import applicationPath, logPath, apiPath, confFileName
+from logger import logResult, mylog, print_log
+from helper import generate_mac_links, removeDuplicateNewLines, timeNowTZ, get_file_content, write_file, get_setting_value, get_timezone_offset
+
+#-------------------------------------------------------------------------------
+# Notification object handling
+#-------------------------------------------------------------------------------
+class Notification_obj:
+    def __init__(self, db):
+        self.db = db
+
+        # Create Notifications table if missing        
+        self.db.sql.execute("""CREATE TABLE IF NOT EXISTS "Notifications" (
+            "Index"           INTEGER,
+            "GUID"            TEXT UNIQUE,
+            "DateTimeCreated" TEXT,
+            "DateTimePushed"  TEXT,
+            "Status"          TEXT,
+            "JSON"            TEXT,
+            "Text"            TEXT,
+            "HTML"            TEXT,
+            "PublishedVia"    TEXT,
+            "Extra"           TEXT,
+            PRIMARY KEY("Index" AUTOINCREMENT)
+        );
+        """)
+
+        self.save()
+
+    # Method to override processing of notifications
+    def on_before_create(self, JSON, Extra):  
+
+        return JSON, Extra
+
+
+    # Create a new DB entry if new notifications available, otherwise skip
+    def create(self, JSON, Extra=""):  
+
+        JSON, Extra = self.on_before_create(JSON, Extra)
+
+        #  Write output data for debug
+        write_file (logPath + '/report_output.json', json.dumps(JSON))
+        
+        # Check if nothing to report, end
+        if JSON["new_devices"] == [] and JSON["down_devices"] == [] and JSON["events"] == [] and JSON["plugins"] == []:
+            self.HasNotifications = False
+        else:            
+            self.HasNotifications = True 
+
+        self.GUID               = str(uuid.uuid4())
+        self.DateTimeCreated    = timeNowTZ()
+        self.DateTimePushed     = ""
+        self.Status             = "new"
+        self.JSON               = JSON
+        self.Text               = ""
+        self.HTML               = ""
+        self.PublishedVia       = ""
+        self.Extra              = Extra
+
+        if self.HasNotifications:
+
+
+            # if not notiStruc.json['data'] and not notiStruc.text and not notiStruc.html:
+            #     mylog('debug', '[Notification] notiStruc is empty')
+            # else:
+            #     mylog('debug', ['[Notification] notiStruc:', json.dumps(notiStruc.__dict__, indent=4)])
+            
+            Text = ""
+            HTML = ""                        
+
+
+            # Open text Template
+            mylog('verbose', ['[Notification] Open text Template'])
+            template_file = open(applicationPath + '/back/report_template.txt', 'r')
+            mail_text = template_file.read()
+            template_file.close()
+
+            # Open html Template
+            mylog('verbose', ['[Notification] Open html Template'])
+
+            # select template type depoending if running latest version or an older one
+            if conf.newVersionAvailable :
+                template_file_path = '/back/report_template_new_version.html'
+            else:
+                template_file_path = '/back/report_template.html'
+
+            mylog('verbose', ['[Notification] Using template', template_file_path])
+            template_file = open(applicationPath + template_file_path, 'r')   
+            mail_html = template_file.read()
+            template_file.close()
+
+
+            # Report "REPORT_DATE" in Header & footer
+            timeFormated = timeNowTZ().strftime ('%Y-%m-%d %H:%M')
+            mail_text = mail_text.replace ('<REPORT_DATE>', timeFormated)
+            mail_html = mail_html.replace ('<REPORT_DATE>', timeFormated)
+
+            # Report "SERVER_NAME" in Header & footer
+            mail_text = mail_text.replace ('<SERVER_NAME>', socket.gethostname() )
+            mail_html = mail_html.replace ('<SERVER_NAME>', socket.gethostname() )
+
+            # Report "VERSION" in Header & footer
+            VERSIONFILE = subprocess.check_output(['php', applicationPath + '/front/php/templates/version.php']).decode('utf-8')
+            mail_text = mail_text.replace ('<BUILD_VERSION>', VERSIONFILE)
+            mail_html = mail_html.replace ('<BUILD_VERSION>', VERSIONFILE)	
+
+            # Report "BUILD" in Header & footer
+            BUILDFILE = subprocess.check_output(['php', applicationPath + '/front/php/templates/build.php']).decode('utf-8')
+            mail_text = mail_text.replace ('<BUILD_DATE>', BUILDFILE)
+            mail_html = mail_html.replace ('<BUILD_DATE>', BUILDFILE)
+
+            # Start generating the TEXT & HTML notification messages
+            html, text = construct_notifications(self.JSON, "new_devices")
+
+            mail_text = mail_text.replace ('<NEW_DEVICES_TABLE>', text + '\n')
+            mail_html = mail_html.replace ('<NEW_DEVICES_TABLE>', html)
+            mylog('verbose', ['[Notification] New Devices sections done.'])
+
+            html, text = construct_notifications(self.JSON, "down_devices")
+
+
+            mail_text = mail_text.replace ('<DOWN_DEVICES_TABLE>', text + '\n')
+            mail_html = mail_html.replace ('<DOWN_DEVICES_TABLE>', html)
+            mylog('verbose', ['[Notification] Down Devices sections done.'])
+
+            html, text = construct_notifications(self.JSON, "events")           
+            
+
+            mail_text = mail_text.replace ('<EVENTS_TABLE>', text + '\n')
+            mail_html = mail_html.replace ('<EVENTS_TABLE>', html)
+            mylog('verbose', ['[Notification] Events sections done.'])    
+
+
+            html, text = construct_notifications(self.JSON, "plugins")
+
+            mail_text = mail_text.replace ('<PLUGINS_TABLE>', text + '\n')
+            mail_html = mail_html.replace ('<PLUGINS_TABLE>', html)
+            
+            mylog('verbose', ['[Notification] Plugins sections done.'])
+
+            final_text = removeDuplicateNewLines(mail_text)
+
+            # Create clickable MAC links            
+            final_html = generate_mac_links (mail_html, conf.REPORT_DASHBOARD_URL + '/deviceDetails.php?mac=')    
+
+            send_api(self.JSON, mail_text, mail_html)
+
+            #  Write output data for debug            
+            write_file (logPath + '/report_output.txt', final_text)
+            write_file (logPath + '/report_output.html', final_html)
+
+            mylog('minimal', ['[Notification] Udating API files'])
+
+            self.Text               = final_text
+            self.HTML               = final_html
+
+            self.upsert()
+        
+        return self
+
+    # Only updates the status
+    def updateStatus(self, newStatus):
+        self.Status = newStatus
+        self.upsert()
+
+    # Updates the Published properties
+    def updatePublishedVia(self, newPublishedVia):
+        self.PublishedVia = newPublishedVia
+        self.DateTimePushed = timeNowTZ()
+        self.upsert()        
+
+    # create or update a notification 
+    def upsert(self):
+        self.db.sql.execute("""
+            INSERT OR REPLACE INTO Notifications (GUID, DateTimeCreated, DateTimePushed, Status, JSON, Text, HTML, PublishedVia, Extra)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, (self.GUID, self.DateTimeCreated, self.DateTimePushed, self.Status, json.dumps(self.JSON), self.Text, self.HTML, self.PublishedVia, self.Extra))
+
+        self.save()
+
+    # Remove notification object by GUID
+    def remove(self, GUID):
+        # Execute an SQL query to delete the notification with the specified GUID
+        self.db.sql.execute("""
+            DELETE FROM Notifications
+            WHERE GUID = ?
+        """, (GUID,))
+        self.save()
+
+    # Get all with the "new" status
+    def getNew(self):
+        self.db.sql.execute("""
+            SELECT * FROM Notifications
+            WHERE Status = "new"
+        """)
+        return self.db.sql.fetchall()
+
+    # Set all to "processed" status
+    def setAllProcessed(self):
+
+        # Execute an SQL query to update the status of all notifications
+        self.db.sql.execute("""
+            UPDATE Notifications
+            SET Status = "processed"
+            WHERE Status = "new"
+        """)
+
+        self.save()
+
+    def clearPendingEmailFlag(self):
+
+        # Clean Pending Alert Events
+        self.db.sql.execute ("""UPDATE Devices SET dev_LastNotification = ?
+                            WHERE dev_MAC IN (
+                                SELECT eve_MAC FROM Events
+                                    WHERE eve_PendingAlertEmail = 1
+                            )
+                        """, (timeNowTZ(),) )
+
+        self.db.sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
+                                    WHERE eve_PendingAlertEmail = 1 
+                                    AND eve_EventType !='Device Down' """)
+
+        # Clear down events flag after the reporting window passed
+        self.db.sql.execute (f"""UPDATE Events SET eve_PendingAlertEmail = 0
+                                    WHERE eve_PendingAlertEmail = 1 
+                                    AND eve_EventType =='Device Down' 
+                                    AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}')
+                            """)
+
+        # clear plugin events
+        self.db.sql.execute ("DELETE FROM Plugins_Events")
+
+        # DEBUG - print number of rows updated
+        mylog('minimal', ['[Notification] Notifications changes: ', self.db.sql.rowcount])    
+
+        self.save()
+
+    def save(self):
+        # Commit changes
+        self.db.commitDB()
+
+#-------------------------------------------------------------------------------
+# Reporting
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+def construct_notifications(JSON, section):
+
+    jsn         = JSON[section]
+
+    # Return if empty
+    if jsn == []:
+        return '',''
+
+    tableTitle  = JSON[section + "_meta"]["title"]
+    headers     = JSON[section + "_meta"]["columnNames"]
+
+    html = ''
+    text = ''
+
+    table_attributes = {"style" : "border-collapse: collapse; font-size: 12px; color:#70707", "width" : "100%", "cellspacing" : 0, "cellpadding" : "3px", "bordercolor" : "#C0C0C0", "border":"1"}
+    headerProps = "width='120px' style='color:white; font-size: 16px;' bgcolor='#64a0d6' "
+    thProps = "width='120px' style='color:#F0F0F0' bgcolor='#64a0d6' "
+
+    build_direction = "TOP_TO_BOTTOM"
+    text_line = '{}\t{}\n'
+
+
+    if len(jsn) > 0:
+        text = tableTitle + "\n---------\n"
+
+        # Convert a JSON into an HTML table
+        html = convert({"data": jsn}, build_direction=build_direction, table_attributes=table_attributes)
+        
+        # Cleanup the generated HTML table notification
+        html = format_table(html, "data", headerProps, tableTitle).replace('<ul>','<ul style="list-style:none;padding-left:0">').replace("<td>null</td>", "<td></td>")        
+
+        # prepare text-only message
+        for device in jsn:
+            for header in headers:
+                padding = ""
+                if len(header) < 4:
+                    padding = "\t"
+                text += text_line.format ( header + ': ' + padding, device[header])
+            text += '\n'
+
+        #  Format HTML table headers
+        for header in headers:
+            html = format_table(html, header, thProps)
+
+    return html, text
+
+#-------------------------------------------------------------------------------
+def send_api(json_final, mail_text, mail_html):
+        mylog('verbose', ['[Send API] Updating notification_* files in ', apiPath])
+
+        write_file(apiPath + 'notification_text.txt'  , mail_text)
+        write_file(apiPath + 'notification_text.html'  , mail_html)
+        write_file(apiPath + 'notification_json_final.json'  , json.dumps(json_final))
+
+
+
+#-------------------------------------------------------------------------------
+# Replacing table headers
+def format_table (html, thValue, props, newThValue = ''):
+
+    if newThValue == '':
+        newThValue = thValue
+
+    return html.replace("<th>"+thValue+"</th>", "<th "+props+" >"+newThValue+"</th>" )
+
+
+

server/plugin.py

@@ -0,0 +1,803 @@
+import os
+import sqlite3
+import json
+import subprocess
+import datetime
+import base64
+
+from collections import namedtuple
+
+# Register NetAlertX modules
+import conf
+from const import pluginsPath, logPath, applicationPath
+from logger import mylog
+from helper import timeNowTZ,  updateState, get_file_content, write_file, get_setting, get_setting_value
+from api import update_api
+from plugin_utils import logEventStatusCounts, get_plugin_string, get_plugin_setting, print_plugin_info, list_to_csv, combine_plugin_objects, resolve_wildcards_arr, handle_empty, custom_plugin_decoder
+from notification import Notification_obj
+
+
+#-------------------------------------------------------------------------------
+class plugin_param:
+    def __init__(self, param, plugin, db):
+
+        paramValuesCount = 1
+
+        #  Get setting value
+        if param["type"] == "setting":
+            inputValue = get_setting(param["value"])
+
+            if inputValue != None:
+                setVal = inputValue["Value"] # setting value
+                setTyp = inputValue["Type"]  # setting type
+
+                noConversion            = ['text', 'string', 'integer', 'boolean', 'password', 'readonly', 'integer.select', 'text.select', 'integer.checkbox'  ]
+                arrayConversion         = ['text.multiselect', 'list', 'subnets']                 
+                jsonConversion          = ['.template'] 
+
+                mylog('debug', f'[Plugins] setTyp: {setTyp}')
+
+                if '.select' in setTyp or setTyp in arrayConversion:
+                    # store number of returned values
+                    paramValuesCount = len(setVal)
+
+                if setTyp in noConversion:
+                    resolved =  setVal
+                
+                elif setTyp in arrayConversion:
+                    #  make them safely passable to a python or linux script
+                    resolved =  list_to_csv(setVal)
+
+                else:
+                    for item in jsonConversion:
+                        if setTyp.endswith(item):
+                            return json.dumps(setVal)
+                        else:
+                            mylog('none', ['[Plugins] ⚠ ERROR: Parameter not converted.'])  
+                
+
+        #  Get SQL result
+        if param["type"] == "sql":
+            inputValue = db.get_sql_array(param["value"])
+
+            # store number of returned values
+            paramValuesCount = len(inputValue)
+
+            #  make them safely passable to a python or linux script
+            resolved = list_to_csv(inputValue)
+
+        
+        mylog('debug', f'[Plugins] Resolved value: {resolved}')
+
+        #  Handle timeout multiplier if script executes multiple time
+        multiplyTimeout = False
+        if 'timeoutMultiplier' in param and param['timeoutMultiplier']:
+            multiplyTimeout = True
+
+        #  Handle base64 encoding
+        encodeToBase64 = False
+        if 'base64' in param and param['base64']:
+            encodeToBase64 = True
+
+
+        mylog('debug', f'[Plugins] Convert to Base64: {encodeToBase64}')
+        if encodeToBase64:
+            resolved = str(base64.b64encode(resolved.encode('ascii')))
+            mylog('debug', f'[Plugins] base64 value: {resolved}')
+
+
+        self.resolved   = resolved        
+        self.inputValue = inputValue        
+        self.base64     = encodeToBase64                
+        self.name       = param["name"]
+        self.type       = param["type"]
+        self.value      = param["value"]
+        self.paramValuesCount       = paramValuesCount    
+        self.multiplyTimeout        = multiplyTimeout    
+
+#-------------------------------------------------------------------------------
+class plugins_state:
+    def __init__(self, processScan = False):
+        self.processScan        = processScan
+
+#-------------------------------------------------------------------------------
+def run_plugin_scripts(db, runType, pluginsState = plugins_state()):
+
+    # Header
+    updateState("Run: Plugins")
+
+    mylog('debug', ['[Plugins] Check if any plugins need to be executed on run type: ', runType])
+
+    for plugin in conf.plugins:
+
+        shouldRun = False        
+        prefix = plugin["unique_prefix"]
+
+        set = get_plugin_setting(plugin, "RUN")
+        if set != None and set['value'] == runType:
+            if runType != "schedule":
+                shouldRun = True
+            elif  runType == "schedule":
+                # run if overdue scheduled time   
+                # check schedules if any contains a unique plugin prefix matching the current plugin
+                for schd in conf.mySchedules:
+                    if schd.service == prefix:          
+                        # Check if schedule overdue
+                        shouldRun = schd.runScheduleCheck()  
+
+        if shouldRun:            
+            # Header
+            updateState(f"Plugins: {prefix}")
+                        
+            print_plugin_info(plugin, ['display_name'])
+            mylog('debug', ['[Plugins] CMD: ', get_plugin_setting(plugin, "CMD")["value"]])
+            pluginsState = execute_plugin(db, plugin, pluginsState) 
+            #  update last run time
+            if runType == "schedule":
+                for schd in conf.mySchedules:
+                    if schd.service == prefix:          
+                        # note the last time the scheduled plugin run was executed
+                        schd.last_run = timeNowTZ()
+
+    return pluginsState
+
+
+
+
+#-------------------------------------------------------------------------------
+# Executes the plugin command specified in the setting with the function specified as CMD 
+def execute_plugin(db, plugin, pluginsState = plugins_state() ):
+    sql = db.sql  
+
+
+    if pluginsState is None:
+        mylog('debug', ['[Plugins] pluginsState is None'])   
+        pluginsState = plugins_state()
+
+    # ------- necessary settings check  --------
+    set = get_plugin_setting(plugin, "CMD")
+
+    #  handle missing "function":"CMD" setting
+    if set == None:                
+        return pluginsState
+
+    set_CMD = set["value"]
+
+    set = get_plugin_setting(plugin, "RUN_TIMEOUT")
+
+    #  handle missing "function":"<unique_prefix>_TIMEOUT" setting
+    if set == None:   
+        set_RUN_TIMEOUT = 10
+    else:     
+        set_RUN_TIMEOUT = set["value"]         
+
+    #  Prepare custom params    
+    params = []
+
+    if "params" in plugin:
+        for param in plugin["params"]:     
+
+            tempParam = plugin_param(param, plugin, db)
+
+            if tempParam.resolved == None:
+                mylog('none', [f'[Plugins] The parameter "name":"{tempParam.name}" for "value": {tempParam.value} was resolved as None'])
+
+            else:
+                # params.append( [param["name"], resolved] )
+                params.append( [tempParam.name, tempParam.resolved] )
+
+                if tempParam.multiplyTimeout:
+                    
+                    set_RUN_TIMEOUT = set_RUN_TIMEOUT*tempParam.paramValuesCount
+
+                    mylog('debug', [f'[Plugins] The parameter "name":"{param["name"]}" will multiply the timeout {tempParam.paramValuesCount} times. Total timeout: {set_RUN_TIMEOUT}s'])
+    
+    mylog('debug', ['[Plugins] Timeout: ', set_RUN_TIMEOUT]) 
+    
+    # build SQL query parameters to insert into the DB
+    sqlParams = []
+
+    # script 
+    if plugin['data_source'] == 'script':
+        # ------- prepare params --------
+        # prepare command from plugin settings, custom parameters  
+        command = resolve_wildcards_arr(set_CMD.split(), params)        
+
+        # Execute command
+        mylog('verbose', ['[Plugins] Executing: ', set_CMD])
+        mylog('debug',   ['[Plugins] Resolved : ', command])        
+
+        try:
+            # try runnning a subprocess with a forced timeout in case the subprocess hangs
+            output = subprocess.check_output (command, universal_newlines=True,  stderr=subprocess.STDOUT, timeout=(set_RUN_TIMEOUT))
+        except subprocess.CalledProcessError as e:
+            # An error occured, handle it
+            mylog('none', [e.output])
+            mylog('none', ['[Plugins] ⚠ ERROR - enable LOG_LEVEL=debug and check logs'])            
+        except subprocess.TimeoutExpired as timeErr:
+            mylog('none', [f'[Plugins] ⚠ ERROR - TIMEOUT - the plugin {plugin["unique_prefix"]} forcefully terminated as timeout reached. Increase TIMEOUT setting and scan interval.']) 
+
+
+        #  check the last run output
+        # Initialize newLines
+        newLines = []
+
+        # Create the file path
+        file_path = os.path.join(pluginsPath, plugin["code_name"], 'last_result.log')
+
+        # Check if the file exists
+        if os.path.exists(file_path):
+            # File exists, open it and read its contents
+            with open(file_path, 'r+') as f:
+                newLines = f.read().split('\n')
+
+            # if the script produced some outpout, clean it up to ensure it's the correct format        
+            # cleanup - select only lines containing a separator to filter out unnecessary data
+            newLines = list(filter(lambda x: '|' in x, newLines))        
+            
+            for line in newLines:
+                columns = line.split("|")
+                # There has to be always 9 columns
+                if len(columns) == 9:
+                    # Create a tuple containing values to be inserted into the database.
+                    # Each value corresponds to a column in the table in the order of the columns.
+                    # must match the Plugins_Objects and Plugins_Events database tables and can be used as input for the plugin_object_class.
+                    sqlParams.append(
+                        (
+                            0,                          # "Index" placeholder
+                            plugin["unique_prefix"],    # "Plugin" column value from the plugin dictionary
+                            columns[0],                 # "Object_PrimaryID" value from columns list
+                            columns[1],                 # "Object_SecondaryID" value from columns list
+                            'null',                     # Placeholder for "DateTimeCreated" column
+                            columns[2],                 # "DateTimeChanged" value from columns list
+                            columns[3],                 # "Watched_Value1" value from columns list
+                            columns[4],                 # "Watched_Value2" value from columns list
+                            columns[5],                 # "Watched_Value3" value from columns list
+                            columns[6],                 # "Watched_Value4" value from columns list
+                            'not-processed',            #  "Status" column (placeholder)
+                            columns[7],                 # "Extra" value from columns list
+                            'null',                     # Placeholder for "UserData" column
+                            columns[8]                  # "ForeignKey" value from columns list
+                        )
+                    )
+                else:
+                    mylog('none', ['[Plugins] Skipped invalid line in the output: ', line])
+        else:
+            mylog('debug', [f'[Plugins] The file {file_path} does not exist'])             
+    
+    # app-db-query
+    if plugin['data_source'] == 'app-db-query':
+        # replace single quotes wildcards
+        q = set_CMD.replace("{s-quote}", '\'')
+
+        # Execute command
+        mylog('verbose', ['[Plugins] Executing: ', q])
+
+        # set_CMD should contain a SQL query        
+        arr = db.get_sql_array (q) 
+
+        for row in arr:
+            # There has to be always 9 columns
+            if len(row) == 9 and (row[0] in ['','null']) == False :
+                # Create a tuple containing values to be inserted into the database.
+                # Each value corresponds to a column in the table in the order of the columns.
+                # must match the Plugins_Objects and Plugins_Events database tables and can be used as input for the plugin_object_class
+                sqlParams.append(
+                    (
+                        0,                          # "Index" placeholder
+                        plugin["unique_prefix"],    # "Plugin" plugin dictionary
+                        row[0],                     # "Object_PrimaryID" row
+                        handle_empty(row[1]),       # "Object_SecondaryID" column after handling empty values
+                        'null',                     # Placeholder "DateTimeCreated" column
+                        row[2],                     # "DateTimeChanged" row
+                        row[3],                     # "Watched_Value1" row
+                        row[4],                     # "Watched_Value2" row
+                        handle_empty(row[5]),       # "Watched_Value3" column after handling empty values
+                        handle_empty(row[6]),       # "Watched_Value4" column after handling empty values
+                        'not-processed',            #  "Status" column (placeholder)
+                        row[7],                     # "Extra" row
+                        'null',                     # Placeholder "UserData" column
+                        row[8]                      # "ForeignKey" row
+                    )
+                )
+            else:
+                mylog('none', ['[Plugins] Skipped invalid sql result'])
+    
+    # app-db-query
+    if plugin['data_source'] == 'sqlite-db-query':
+        # replace single quotes wildcards
+        # set_CMD should contain a SQL query    
+        q = set_CMD.replace("{s-quote}", '\'')
+
+        # Execute command
+        mylog('verbose', ['[Plugins] Executing: ', q])
+
+        # ------- necessary settings check  --------
+        set = get_plugin_setting(plugin, "DB_PATH")
+
+        #  handle missing "function":"DB_PATH" setting
+        if set == None:                
+            mylog('none', ['[Plugins] ⚠ ERROR: DB_PATH setting for plugin type sqlite-db-query missing.'])
+            return pluginsState
+        
+        fullSqlitePath = set["value"]
+
+
+        #  try attaching the sqlite DB
+        try:
+            sql.execute ("ATTACH DATABASE '"+ fullSqlitePath +"' AS EXTERNAL_"+plugin["unique_prefix"])
+            arr = db.get_sql_array (q) 
+            sql.execute ("DETACH DATABASE EXTERNAL_"+plugin["unique_prefix"])
+             
+        except sqlite3.Error as e:            
+            mylog('none',[f'[Plugins] ⚠ ERROR: DB_PATH setting ({fullSqlitePath}) for plugin {plugin["unique_prefix"]}. Did you mount it correctly?'])
+            mylog('none',[f'[Plugins] ⚠ ERROR: ATTACH DATABASE failed with SQL ERROR: ', e])
+            return pluginsState            
+
+        for row in arr:
+            # There has to be always 9 columns
+            if len(row) == 9 and (row[0] in ['','null']) == False :
+                # Create a tuple containing values to be inserted into the database.
+                # Each value corresponds to a column in the table in the order of the columns.
+                # must match the Plugins_Objects and Plugins_Events database tables and can be used as input for the plugin_object_class
+                sqlParams.append((
+                    0,                            #  "Index" placeholder
+                    plugin["unique_prefix"],      #  "Plugin" 
+                    row[0],                       #  "Object_PrimaryID" 
+                    handle_empty(row[1]),         #  "Object_SecondaryID" 
+                    'null',                       #  "DateTimeCreated" column (null placeholder)
+                    row[2],                       #  "DateTimeChanged" 
+                    row[3],                       #  "Watched_Value1" 
+                    row[4],                       #  "Watched_Value2" 
+                    handle_empty(row[5]),         #  "Watched_Value3" 
+                    handle_empty(row[6]),         #  "Watched_Value4" 
+                    'not-processed',              #  "Status" column (placeholder)
+                    row[7],                       #  "Extra" 
+                    'null',                       #  "UserData" column (null placeholder)
+                    row[8]))                      #  "ForeignKey" 
+            else:
+                mylog('none', ['[Plugins] Skipped invalid sql result'])
+
+    
+    # check if the subprocess / SQL query failed / there was no valid output
+    if len(sqlParams) == 0: 
+        mylog('none', ['[Plugins] No output received from the plugin ', plugin["unique_prefix"], ' - enable LOG_LEVEL=debug and check logs'])
+        return pluginsState 
+    else: 
+        mylog('verbose', ['[Plugins] SUCCESS, received ', len(sqlParams), ' entries'])  
+        mylog('debug',   ['[Plugins] sqlParam entries: ', sqlParams])
+
+    # process results if any
+    if len(sqlParams) > 0:               
+
+        # create objects
+        pluginsState = process_plugin_events(db, plugin, pluginsState, sqlParams)
+
+        # update API endpoints
+        update_api(db, False, ["plugins_events","plugins_objects", "plugins_history", "appevents"])  
+    
+    return pluginsState
+
+
+
+
+
+#-------------------------------------------------------------------------------
+# Check if watched values changed for the given plugin
+def process_plugin_events(db, plugin, pluginsState, plugEventsArr):    
+      
+    sql = db.sql
+
+    # Access the connection from the DB instance
+    conn = db.sql_connection
+
+    pluginPref = plugin["unique_prefix"]
+
+    mylog('debug', ['[Plugins] Processing        : ', pluginPref])
+
+    try:
+        # Begin a transaction
+        with conn:
+
+            pluginObjects = []
+            pluginEvents  = []
+
+            #  Create plugin objects from existing database entries
+            plugObjectsArr = db.get_sql_array ("SELECT * FROM Plugins_Objects where Plugin = '" + str(pluginPref)+"'") 
+
+            for obj in plugObjectsArr: 
+                pluginObjects.append(plugin_object_class(plugin, obj))
+
+           
+            # create plugin objects from events - will be processed to find existing objects 
+            for eve in plugEventsArr:                                  
+                pluginEvents.append(plugin_object_class(plugin, eve))
+
+
+            mylog('debug', ['[Plugins] Existing objects from Plugins_Objects: ', len(pluginObjects)])
+            mylog('debug', ['[Plugins] Logged events from the plugin run    : ', len(pluginEvents)])
+
+            
+            #  Loop thru all current events and update the status to "exists" if the event matches an existing object
+            index = 0
+            for tmpObjFromEvent in pluginEvents:        
+
+                #  compare hash of the IDs for uniqueness
+                if any(x.idsHash == tmpObjFromEvent.idsHash for x in pluginObjects):                    
+            
+                    pluginEvents[index].status = "exists"            
+                index += 1
+
+
+            # Loop thru events and check if the ones that exist have changed in the watched columns
+            # if yes update status accordingly
+            index = 0
+            for tmpObjFromEvent in pluginEvents:  
+
+                if tmpObjFromEvent.status == "exists": 
+
+                    #  compare hash of the changed watched columns for uniqueness
+                    if any(x.watchedHash != tmpObjFromEvent.watchedHash for x in pluginObjects):
+                        pluginEvents[index].status = "watched-changed"                            
+                    else:
+                        pluginEvents[index].status = "watched-not-changed"  
+                index += 1
+
+            # Loop thru events and check if previously available objects are missing
+            for tmpObj in pluginObjects:
+                
+                isMissing = True
+
+                for tmpObjFromEvent in pluginEvents:
+                    if tmpObj.idsHash == tmpObjFromEvent.idsHash:
+                        isMissing = False
+
+                if isMissing:
+                    # if wasn't missing before, mark as changed
+                    if tmpObj.status != "missing-in-last-scan":
+                        tmpObj.changed = timeNowTZ().strftime('%Y-%m-%d %H:%M:%S')
+                        tmpObj.status = "missing-in-last-scan"                    
+                    # mylog('debug', [f'[Plugins] Missing from last scan (PrimaryID | SecondaryID): {tmpObj.primaryId} | {tmpObj.secondaryId}'])
+
+
+            # Merge existing plugin objects with newly discovered ones and update existing ones with new values
+            for tmpObjFromEvent in pluginEvents:
+                # set "new" status for new objects and append
+                if tmpObjFromEvent.status == 'not-processed':
+                    # This is a new object as it was not discovered as "exists" previously                    
+                    tmpObjFromEvent.status = 'new'
+
+                    pluginObjects.append(tmpObjFromEvent)
+                # update data of existing objects
+                else:
+                    index = 0
+                    for plugObj in pluginObjects:
+                        # find corresponding object for the event and merge
+                        if plugObj.idsHash == tmpObjFromEvent.idsHash:               
+                            pluginObjects[index] =  combine_plugin_objects(plugObj, tmpObjFromEvent)                    
+
+                        index += 1
+
+            # Update the DB
+            # ----------------------------
+            # Update the Plugin_Objects    
+            # Create lists to hold the data for bulk insertion
+            objects_to_insert = []
+            events_to_insert = []
+            history_to_insert = []
+            objects_to_update = []
+
+            # only generate events that we want to be notified on (we only need to do this once as all plugObj have the same prefix)
+            statuses_to_report_on = get_setting_value(pluginPref + "_REPORT_ON")  
+            
+            for plugObj in pluginObjects:
+                #  keep old createdTime time if the plugObj already was created before
+                createdTime = plugObj.changed if plugObj.status == 'new' else plugObj.created
+                #  13 values without Index
+                values = (
+                    plugObj.pluginPref, plugObj.primaryId, plugObj.secondaryId, createdTime,
+                    plugObj.changed, plugObj.watched1, plugObj.watched2, plugObj.watched3,
+                    plugObj.watched4, plugObj.status, plugObj.extra, plugObj.userData,
+                    plugObj.foreignKey
+                )
+
+                if plugObj.status == 'new':
+                    objects_to_insert.append(values)
+                else:
+                    objects_to_update.append(values + (plugObj.index,))  # Include index for UPDATE              
+                
+                if plugObj.status in statuses_to_report_on:
+                    events_to_insert.append(values)
+
+                # combine all DB insert and update events into one for history
+                history_to_insert.append(values)
+
+            
+            mylog('debug', ['[Plugins] pluginEvents      count: ', len(pluginEvents)])
+            mylog('debug', ['[Plugins] pluginObjects     count: ', len(pluginObjects)])
+
+            mylog('debug', ['[Plugins] events_to_insert  count: ', len(events_to_insert)])
+            mylog('debug', ['[Plugins] history_to_insert count: ', len(history_to_insert)])
+            mylog('debug', ['[Plugins] objects_to_insert count: ', len(objects_to_insert)])
+            mylog('debug', ['[Plugins] objects_to_update count: ', len(objects_to_update)])
+
+            logEventStatusCounts('pluginEvents', pluginEvents)
+            logEventStatusCounts('pluginObjects', pluginObjects)
+
+            # Bulk insert objects
+            if objects_to_insert:
+                sql.executemany(
+                    """
+                    INSERT INTO Plugins_Objects 
+                    ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", 
+                    "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", 
+                    "Watched_Value4", "Status", "Extra", "UserData", "ForeignKey") 
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """, objects_to_insert
+                )
+
+            # Bulk update objects
+            if objects_to_update:
+                sql.executemany(
+                    """
+                    UPDATE Plugins_Objects
+                    SET "Plugin" = ?, "Object_PrimaryID" = ?, "Object_SecondaryID" = ?, "DateTimeCreated" = ?, 
+                        "DateTimeChanged" = ?, "Watched_Value1" = ?, "Watched_Value2" = ?, "Watched_Value3" = ?, 
+                        "Watched_Value4" = ?, "Status" = ?, "Extra" = ?, "UserData" = ?, "ForeignKey" = ?
+                    WHERE "Index" = ?
+                    """, objects_to_update
+                )
+
+            # Bulk insert events
+            if events_to_insert:
+
+                sql.executemany(
+                    """
+                    INSERT INTO Plugins_Events 
+                    ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", 
+                    "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", 
+                    "Watched_Value4", "Status", "Extra", "UserData", "ForeignKey") 
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """, events_to_insert
+                )
+
+            # Bulk insert history entries
+            if history_to_insert:
+
+                sql.executemany(
+                    """
+                    INSERT INTO Plugins_History 
+                    ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", 
+                    "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", 
+                    "Watched_Value4", "Status", "Extra", "UserData", "ForeignKey") 
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                    """, history_to_insert
+                )
+
+            # Commit changes to the database
+            db.commitDB()
+
+
+
+    except Exception as e:
+        # Rollback the transaction in case of an error
+        conn.rollback()
+        mylog('none', ['[Plugins] ⚠ ERROR: ', e])
+        raise e   
+
+    # Perform database table mapping if enabled for the plugin
+    if len(pluginEvents) > 0 and "mapped_to_table" in plugin:
+
+        # Initialize an empty list to store SQL parameters.
+        sqlParams = []
+
+        # Get the database table name from the 'mapped_to_table' key in the 'plugin' dictionary.
+        dbTable = plugin['mapped_to_table']
+
+        # Log a debug message indicating the mapping of objects to the database table.
+        mylog('debug', ['[Plugins] Mapping objects to database table: ', dbTable])
+
+        # Initialize lists to hold mapped column names, columnsStr, and valuesStr for SQL query.
+        mappedCols = []
+        columnsStr = ''
+        valuesStr = ''
+
+        # Loop through the 'database_column_definitions' in the 'plugin' dictionary to collect mapped columns.
+        # Build the columnsStr and valuesStr for the SQL query.
+        for clmn in plugin['database_column_definitions']:
+            if 'mapped_to_column' in clmn:
+                mappedCols.append(clmn)
+
+                columnsStr = f'{columnsStr}, "{clmn["mapped_to_column"]}"'
+                valuesStr = f'{valuesStr}, ?'
+
+        # Remove the first ',' from columnsStr and valuesStr.
+        if len(columnsStr) > 0:
+            columnsStr = columnsStr[1:]
+            valuesStr = valuesStr[1:]
+
+        # Map the column names to plugin object event values and create a list of tuples 'sqlParams'.
+        for plgEv in pluginEvents:
+            tmpList = []
+
+            for col in mappedCols:
+                if col['column'] == 'Index':
+                    tmpList.append(plgEv.index)
+                elif col['column'] == 'Plugin':
+                    tmpList.append(plgEv.pluginPref)
+                elif col['column'] == 'Object_PrimaryID':
+                    tmpList.append(plgEv.primaryId)
+                elif col['column'] == 'Object_SecondaryID':
+                    tmpList.append(plgEv.secondaryId)
+                elif col['column'] == 'DateTimeCreated':
+                    tmpList.append(plgEv.created)
+                elif col['column'] == 'DateTimeChanged':
+                    tmpList.append(plgEv.changed)
+                elif col['column'] == 'Watched_Value1':
+                    tmpList.append(plgEv.watched1)
+                elif col['column'] == 'Watched_Value2':
+                    tmpList.append(plgEv.watched2)
+                elif col['column'] == 'Watched_Value3':
+                    tmpList.append(plgEv.watched3)
+                elif col['column'] == 'Watched_Value4':
+                    tmpList.append(plgEv.watched4)
+                elif col['column'] == 'UserData':
+                    tmpList.append(plgEv.userData)
+                elif col['column'] == 'Extra':
+                    tmpList.append(plgEv.extra)
+                elif col['column'] == 'Status':
+                    tmpList.append(plgEv.status)
+
+                # Check if there's a default value specified for this column in the JSON.
+                if 'mapped_to_column_data' in col and 'value' in col['mapped_to_column_data']:
+                    tmpList.append(col['mapped_to_column_data']['value'])
+                
+            # Append the mapped values to the list 'sqlParams' as a tuple.
+            sqlParams.append(tuple(tmpList))
+
+        # Generate the SQL INSERT query using the collected information.
+        q = f'INSERT into {dbTable} ({columnsStr}) VALUES ({valuesStr})'
+
+        # Log a debug message showing the generated SQL query for mapping.
+        mylog('debug', ['[Plugins] SQL query for mapping: ', q])
+        mylog('debug', ['[Plugins] SQL sqlParams for mapping: ', sqlParams])
+
+        # Execute the SQL query using 'sql.executemany()' and the 'sqlParams' list of tuples.
+        # This will insert multiple rows into the database in one go.
+        sql.executemany(q, sqlParams)
+
+        db.commitDB()
+
+        # perform scan if mapped to CurrentScan table
+        if dbTable == 'CurrentScan':
+            pluginsState.processScan = True
+
+    db.commitDB()
+
+
+    return pluginsState
+
+
+#-------------------------------------------------------------------------------
+class plugin_object_class:
+    def __init__(self, plugin, objDbRow):
+        self.index        = objDbRow[0]
+        self.pluginPref   = objDbRow[1]
+        self.primaryId    = objDbRow[2]
+        self.secondaryId  = objDbRow[3]
+        self.created      = objDbRow[4] # can be null
+        self.changed      = objDbRow[5] # never null (data coming from plugin)
+        self.watched1     = objDbRow[6]
+        self.watched2     = objDbRow[7]
+        self.watched3     = objDbRow[8]
+        self.watched4     = objDbRow[9]
+        self.status       = objDbRow[10]
+        self.extra        = objDbRow[11]
+        self.userData     = objDbRow[12]
+        self.foreignKey   = objDbRow[13]
+
+        # Check if self.status is valid
+        if self.status not in ["exists", "watched-changed", "watched-not-changed", "new", "not-processed", "missing-in-last-scan"]:
+            raise ValueError("Invalid status value for plugin object:", self.status)
+
+        self.idsHash      = str(hash(str(self.primaryId) + str(self.secondaryId)))    
+        # self.idsHash      = str(self.primaryId) + str(self.secondaryId)
+
+        self.watchedClmns = []
+        self.watchedIndxs = []          
+
+        setObj = get_plugin_setting(plugin, 'WATCH')
+
+        indexNameColumnMapping = [(6, 'Watched_Value1' ), (7, 'Watched_Value2' ), (8, 'Watched_Value3' ), (9, 'Watched_Value4' )]
+
+        if setObj is not None:            
+            
+            self.watchedClmns = setObj["value"]
+
+            for clmName in self.watchedClmns:
+                for mapping in indexNameColumnMapping:
+                    if clmName == indexNameColumnMapping[1]:
+                        self.watchedIndxs.append(indexNameColumnMapping[0])
+
+        tmp = ''
+        for indx in self.watchedIndxs:
+            tmp += str(objDbRow[indx])
+
+        self.watchedHash  = str(hash(tmp))
+
+
+#===============================================================================
+# Handling of  user initialized front-end events
+#===============================================================================
+def check_and_run_user_event(db, pluginsState):
+    # Check if the log file exists
+    logFile = os.path.join(logPath, "execution_queue.log")
+    
+    if not os.path.exists(logFile):
+        return pluginsState
+
+    with open(logFile, "r") as file:
+        lines = file.readlines()
+
+    for line in lines:
+        # Split the line by '|', and take the third and fourth columns (indices 2 and 3)
+        columns = line.strip().split('|')[2:4]
+
+        if len(columns) != 2:
+            continue
+
+        event, param = columns
+
+        if event == 'test':
+            pluginsState = handle_test(param, db, pluginsState)
+        if event == 'run':
+            pluginsState = handle_run(param, db, pluginsState)
+        if event == 'update_api':
+            # update API endpoints
+            update_api(db, False, param.split(','))  
+
+    # Clear the log file
+    open(logFile, "w").close()
+
+    return pluginsState
+
+
+#-------------------------------------------------------------------------------
+def handle_run(runType, db, pluginsState):
+    
+    mylog('minimal', ['[', timeNowTZ(), '] START Run: ', runType])
+    
+    # run the plugin to run
+    for plugin in conf.plugins:
+        if plugin["unique_prefix"] == runType:                
+            pluginsState = execute_plugin(db, plugin, pluginsState) 
+
+    mylog('minimal', ['[', timeNowTZ(), '] END Run: ', runType])
+    return pluginsState
+
+
+
+#-------------------------------------------------------------------------------
+def handle_test(runType, db, pluginsState):
+
+    mylog('minimal', ['[', timeNowTZ(), '] [Test] START Test: ', runType])
+    
+    # Prepare test samples
+    sample_txt = get_file_content(applicationPath + '/back/report_sample.txt')    
+    sample_html = get_file_content(applicationPath + '/back/report_sample.html')    
+    sample_json = json.loads(get_file_content(applicationPath + '/back/webhook_json_sample.json'))[0]["body"]["attachments"][0]["text"]
+    
+    # Create fake notification
+    notification    = Notification_obj(db)
+    notificationObj = notification.create(sample_json, "")
+
+    # Run test
+    pluginsState = handle_run(runType, db, pluginsState)
+
+    # Remove sample notification
+    notificationObj.remove(notificationObj.GUID)    
+
+    mylog('minimal', ['[Test] END Test: ', runType])
+
+    return pluginsState
+

server/plugin_utils.py

@@ -0,0 +1,231 @@
+import os
+import json
+
+import conf 
+from logger import mylog
+from const import pluginsPath, logPath, apiPath
+from helper import timeNowTZ,  updateState, get_file_content, write_file, get_setting, get_setting_value
+
+module_name = 'Plugin utils'
+
+#-------------------------------------------------------------------------------
+def logEventStatusCounts(objName, pluginEvents):
+    status_counts = {}  # Dictionary to store counts for each status
+
+    for event in pluginEvents:
+        status = event.status
+        if status in status_counts:
+            status_counts[status] += 1
+        else:
+            status_counts[status] = 1
+
+    for status, count in status_counts.items():
+        mylog('debug', [f'[{module_name}] In {objName} there are {count} events with the status "{status}" '])
+
+
+#-------------------------------------------------------------------------------
+def print_plugin_info(plugin, elements = ['display_name']):
+
+    mylog('verbose', [f'[{module_name}] ---------------------------------------------']) 
+
+    for el in elements:
+        res = get_plugin_string(plugin, el)
+        mylog('verbose', [f'[{module_name}] ', el ,': ', res]) 
+
+
+#-------------------------------------------------------------------------------
+# Gets the whole setting object
+def get_plugin_setting(plugin, function_key):
+    
+    result = None
+
+    for set in plugin['settings']:
+        if set["function"] == function_key:
+          result =  set 
+          
+    # if result == None:
+    #     mylog('debug', [f'[{module_name}] Setting with "function":"', function_key, '" is missing in plugin: ', get_plugin_string(plugin, 'display_name')])
+
+    return result
+
+
+        
+
+#-------------------------------------------------------------------------------
+# Get localized string value on the top JSON depth, not recursive
+def get_plugin_string(props, el):
+
+    result = ''
+
+    if el in props['localized']:
+        for val in props[el]:
+            if val['language_code'] == 'en_us':
+                result = val['string']
+        
+        if result == '':
+            result = 'en_us string missing'
+
+    else:
+        result = props[el]
+    
+    return result
+
+
+#-------------------------------------------------------------------------------
+#  generates a comma separated list of values from a list (or a string representing a list)
+def list_to_csv(arr):
+    tmp = ''
+    arrayItemStr = ''
+
+    mylog('debug', f'[{module_name}] Flattening the below array')    
+    mylog('debug', arr)    
+    mylog('debug', f'[{module_name}] isinstance(arr, list) : {isinstance(arr, list)} | isinstance(arr, str) : {isinstance(arr, str)}')    
+
+    if isinstance(arr, str):
+        tmpStr =  arr.replace('[','').replace(']','').replace("'", '')  # removing brackets and single quotes (not allowed)    
+        
+        if ',' in tmpStr:            
+            # Split the string into a list and trim whitespace
+            cleanedStr = [tmpSubStr.strip() for tmpSubStr in tmpStr.split(',')]
+
+            # Join the list elements using a comma
+            result_string = ",".join(cleanedStr)   
+        else:
+            result_string = tmpStr
+
+        return result_string
+
+    elif isinstance(arr, list):
+        for arrayItem in arr:
+            # only one column flattening is supported
+            if isinstance(arrayItem, list):  
+                arrayItemStr = str(arrayItem[0]).replace("'", '').strip()  # removing single quotes - not allowed            
+            else:
+                # is string already
+                arrayItemStr = arrayItem
+
+
+            tmp += f'{arrayItemStr},'
+
+        tmp = tmp[:-1]  # Remove last comma ','
+
+        mylog('debug', f'[{module_name}] Flattened array: {tmp}')    
+
+        return tmp
+
+    else:
+        mylog('none', f'[{module_name}] ⚠ ERROR Could not convert array: {arr}')    
+
+
+
+
+#-------------------------------------------------------------------------------
+# Combine plugin objects, keep user-defined values, created time, changed time if nothing changed and the index
+def combine_plugin_objects(old, new):    
+    
+    new.userData = old.userData 
+    new.index = old.index 
+    new.created = old.created 
+
+    # Keep changed time if nothing changed
+    if new.status in ['watched-not-changed']:
+        new.changed = old.changed
+
+    #  return the new object, with some of the old values
+    return new
+
+
+
+#-------------------------------------------------------------------------------
+# Replace {wildcars} with parameters
+def resolve_wildcards_arr(commandArr, params):
+
+    mylog('debug', [f'[{module_name}] Pre-Resolved CMD: '] + commandArr)   
+
+    for param in params:
+        # mylog('debug', ['[Plugins] key     : {', param[0], '}'])
+        # mylog('debug', ['[Plugins] resolved: ', param[1]])
+
+        i = 0
+        
+        for comPart in commandArr:
+
+            commandArr[i] = comPart.replace('{' + str(param[0]) + '}', str(param[1])).replace('{s-quote}',"'")
+
+            i += 1
+
+    return commandArr   
+
+
+
+#-------------------------------------------------------------------------------
+def get_plugins_configs():
+    pluginsList = []  # Create an empty list to store plugin configurations
+    
+    # Get a list of top-level directories in the specified pluginsPath
+    dirs = next(os.walk(pluginsPath))[1]
+    
+    # Loop through each directory (plugin folder) in dirs
+    for d in dirs:
+        # Check if the directory name does not start with "__" to skip python cache
+        if not d.startswith("__"):
+            # Check if the 'ignore_plugin' file exists in the plugin folder
+            ignore_plugin_path = os.path.join(pluginsPath, d, "ignore_plugin")
+            if not os.path.isfile(ignore_plugin_path):
+                # Construct the path to the config.json file within the plugin folder
+                config_path = os.path.join(pluginsPath, d, "config.json")
+                
+                # Load the contents of the config.json file as a JSON object and append it to pluginsList
+                pluginsList.append(json.loads(get_file_content(config_path)))
+    
+    return pluginsList  # Return the list of plugin configurations
+
+
+#-------------------------------------------------------------------------------
+def custom_plugin_decoder(pluginDict):
+    return namedtuple('X', pluginDict.keys())(*pluginDict.values())        
+
+#-------------------------------------------------------------------------------
+# Handle empty value 
+def handle_empty(value):    
+    if value == '' or value is None:
+        value = 'null'
+
+    return value    
+
+#-------------------------------------------------------------------------------
+# Get and return a plugin object based on key-value pairs
+# keyValues example: getPluginObject({"Plugin":"MQTT", "Watched_Value4":"someValue"})
+def getPluginObject(keyValues):
+
+    plugins_objects = apiPath + 'table_plugins_objects.json'
+
+    try:
+        with open(plugins_objects, 'r') as json_file:
+            data = json.load(json_file)
+
+            objectEntries = data.get("data", [])            
+
+            for item in objectEntries:
+                # Initialize a flag to check if all key-value pairs match
+                all_match = True                
+
+                for key, value in keyValues.items():
+                    if item.get(key) != value:
+                        all_match = False
+                        break  # No need to continue checking if one pair doesn't match
+                
+                if all_match:
+                    return item
+
+            mylog('verbose', [f'[{module_name}] 💬 INFO - Object not found {json.dumps(keyValues)} '])  
+
+            return {}
+
+    except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
+        # Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format
+        mylog('verbose', [f'[{module_name}] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {plugins_objects}'])                
+
+        return {}
+
+

server/reporting.py

@@ -0,0 +1,183 @@
+#---------------------------------------------------------------------------------#
+#  NetAlertX                                                                       #
+#  Open Source Network Guard / WIFI & LAN intrusion detector                      #  
+#                                                                                 #
+#  reporting.py - NetAlertX Back module. Template to email reporting in HTML format #
+#---------------------------------------------------------------------------------#
+#    Puche      2021        pi.alert.application@gmail.com   GNU GPLv3            #
+#    jokob-sk   2022        jokob.sk@gmail.com               GNU GPLv3            #
+#    leiweibau  2022        https://github.com/leiweibau     GNU GPLv3            #
+#    cvc90      2023        https://github.com/cvc90         GNU GPLv3            #
+#---------------------------------------------------------------------------------#
+
+import datetime
+import json
+
+import conf
+from const import applicationPath, logPath, apiPath, confFileName
+from helper import timeNowTZ, get_file_content, write_file, get_timezone_offset, get_setting_value
+from logger import logResult, mylog, print_log
+
+
+#===============================================================================
+# REPORTING
+#===============================================================================
+
+
+#-------------------------------------------------------------------------------
+def get_notifications (db):
+
+    sql = db.sql  #TO-DO
+    
+    # Reporting section
+    mylog('verbose', ['[Notification] Check if something to report'])
+
+    # prepare variables for JSON construction    
+    json_new_devices = []
+    json_new_devices_meta = {}
+    json_down_devices = []
+    json_down_devices_meta = {}
+    json_events = []    
+    json_events_meta = {}
+    json_plugins = []
+    json_plugins_meta = {}
+
+    # Disable reporting on events for devices where reporting is disabled based on the MAC address
+    sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
+                    WHERE eve_PendingAlertEmail = 1 AND eve_EventType != 'Device Down' AND eve_MAC IN
+                        (
+                            SELECT dev_MAC FROM Devices WHERE dev_AlertEvents = 0
+						)""")
+    sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
+                    WHERE eve_PendingAlertEmail = 1 AND eve_EventType = 'Device Down' AND eve_MAC IN
+                        (
+                            SELECT dev_MAC FROM Devices WHERE dev_AlertDeviceDown = 0
+						)""")
+    
+    sections = get_setting_value('NTFPRCS_INCLUDED_SECTIONS')
+
+    mylog('verbose', ['[Notification] Included sections: ', sections ])
+
+    if 'new_devices' in sections:
+        # Compose New Devices Section
+        sqlQuery = f"""SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments  FROM Events_Devices
+                        WHERE eve_PendingAlertEmail = 1
+                        AND eve_EventType = 'New Device' 
+                        {get_setting_value('NTFPRCS_new_dev_condition').replace('{s-quote}',"'")} 
+                        ORDER BY eve_DateTime"""   
+
+        mylog('debug', ['[Notification] new_devices SQL query: ', sqlQuery ])
+
+        # Get the events as JSON
+        json_obj = db.get_table_as_json(sqlQuery)
+
+        json_new_devices_meta = {
+            "title": "New devices",
+            "columnNames": json_obj.columnNames
+        }
+
+        json_new_devices = json_obj.json["data"]    
+
+    if 'down_devices' in sections:
+        # Compose Devices Down Section 
+        # - select only Down Alerts with pending email of devices that didn't reconnect within the specified time window
+        sqlQuery = f"""
+                    SELECT dev_Name, eve_MAC, dev_Vendor, eve_IP, eve_DateTime, eve_EventType  
+                        FROM Events_Devices AS down_events
+                        WHERE eve_PendingAlertEmail = 1 
+                        AND down_events.eve_EventType = 'Device Down' 
+                        AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}')
+                        AND NOT EXISTS (
+                            SELECT 1
+                            FROM Events AS connected_events
+                            WHERE connected_events.eve_MAC = down_events.eve_MAC
+                                AND connected_events.eve_EventType = 'Connected'
+                                AND connected_events.eve_DateTime > down_events.eve_DateTime        
+                        )
+                        ORDER BY down_events.eve_DateTime;
+                    """
+        
+        # Get the events as JSON        
+        json_obj = db.get_table_as_json(sqlQuery)
+
+        json_down_devices_meta = {
+            "title": "Down devices",
+            "columnNames": json_obj.columnNames
+        }
+        json_down_devices = json_obj.json["data"]     
+
+    if 'events' in sections:
+        # Compose Events Section
+        sqlQuery = f"""SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments  FROM Events_Devices
+                        WHERE eve_PendingAlertEmail = 1
+                        AND eve_EventType IN ('Connected','Disconnected',
+                            'IP Changed') 
+                            {get_setting_value('NTFPRCS_event_condition').replace('{s-quote}',"'")} 
+                        ORDER BY eve_DateTime"""      
+
+        mylog('debug', ['[Notification] events SQL query: ', sqlQuery ])
+        
+        # Get the events as JSON        
+        json_obj = db.get_table_as_json(sqlQuery)
+
+        json_events_meta = {
+            "title": "Events",
+            "columnNames": json_obj.columnNames
+        }
+        json_events = json_obj.json["data"]     
+
+    if 'plugins' in sections:
+        # Compose Plugins Section
+        sqlQuery = """SELECT Plugin, Object_PrimaryId, Object_SecondaryId, DateTimeChanged, Watched_Value1, Watched_Value2, Watched_Value3, Watched_Value4, Status from Plugins_Events"""        
+        
+        # Get the events as JSON        
+        json_obj = db.get_table_as_json(sqlQuery)
+
+        json_plugins_meta = {
+            "title": "Plugins",
+            "columnNames": json_obj.columnNames
+        }
+        json_plugins = json_obj.json["data"]   
+
+
+    final_json = {                    
+                    "new_devices": json_new_devices,
+                    "new_devices_meta": json_new_devices_meta,
+                    "down_devices": json_down_devices,
+                    "down_devices_meta": json_down_devices_meta,
+                    "events": json_events,                    
+                    "events_meta": json_events_meta,                    
+                    "plugins": json_plugins,
+                    "plugins_meta": json_plugins_meta,
+                }
+
+    return final_json
+
+
+
+#-------------------------------------------------------------------------------
+def skip_repeated_notifications (db):
+
+    # Skip repeated notifications
+    # due strfime : Overflow --> use  "strftime / 60"
+    mylog('verbose','[Skip Repeated Notifications] Skip Repeated')
+    
+    db.sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
+                    WHERE eve_PendingAlertEmail = 1 AND eve_MAC IN
+                        (
+                        SELECT dev_MAC FROM Devices
+                        WHERE dev_LastNotification IS NOT NULL
+                          AND dev_LastNotification <>""
+                          AND (strftime("%s", dev_LastNotification)/60 +
+                                dev_SkipRepeated * 60) >
+                              (strftime('%s','now','localtime')/60 )
+                        )
+                 """ )
+   
+
+    db.commitDB()
+
+
+
+
+

server/scheduler.py

@@ -0,0 +1,43 @@
+""" class to manage schedules """
+import datetime
+
+from logger import mylog, print_log
+import conf
+
+#-------------------------------------------------------------------------------
+class schedule_class:
+    def __init__(self, service, scheduleObject, last_next_schedule, was_last_schedule_used, last_run = 0):
+        self.service = service
+        self.scheduleObject = scheduleObject
+        self.last_next_schedule = last_next_schedule
+        self.last_run = last_run
+        self.was_last_schedule_used = was_last_schedule_used         
+         
+    def runScheduleCheck(self):
+
+        result = False 
+
+        # Initialize the last run time if never run before
+        if self.last_run == 0:
+            self.last_run =  (datetime.datetime.now(conf.tz) - datetime.timedelta(days=365)).replace(microsecond=0)
+
+        # get the current time with the currently specified timezone
+        nowTime = datetime.datetime.now(conf.tz).replace(microsecond=0)
+
+        # Run the schedule if the current time is past the schedule time we saved last time and 
+        #               (maybe the following check is unnecessary)                
+        if nowTime > self.last_next_schedule:
+            mylog('debug',f'[Scheduler] - Scheduler run for {self.service}: YES')
+            self.was_last_schedule_used = True
+            result = True
+        else:
+            mylog('debug',f'[Scheduler] - Scheduler run for {self.service}: NO')
+            # mylog('debug',f'[Scheduler] - nowTime {nowTime}')
+            # mylog('debug',f'[Scheduler] - self.last_next_schedule {self.last_next_schedule}')
+            # mylog('debug',f'[Scheduler] - self.last_run {self.last_run}')
+        
+        if self.was_last_schedule_used:
+            self.was_last_schedule_used = False
+            self.last_next_schedule = self.scheduleObject.next()            
+
+        return result