From 750fb33e1ccf2b2d20dcd507df05e1f2cad7d7e4 Mon Sep 17 00:00:00 2001 From: Ingo Ratsdorf Date: Mon, 15 Sep 2025 15:54:51 +1200 Subject: [PATCH 1/3] clearPluginObjects added sub to be called during main loop to clear plugins_objects table --- server/__main__.py | 1 + server/models/notification_instance.py | 213 +++++++++++++------------ 2 files changed, 111 insertions(+), 103 deletions(-) diff --git a/server/__main__.py b/server/__main__.py index 426727e6..a8369299 100755 --- a/server/__main__.py +++ b/server/__main__.py @@ -189,6 +189,7 @@ def main (): notification.clearPendingEmailFlag() else: + notification.clearPluginObjects() mylog('verbose', ['[Notification] No changes to report']) # Commit SQL diff --git a/server/models/notification_instance.py b/server/models/notification_instance.py index 1bb82744..403cbd8e 100755 --- a/server/models/notification_instance.py +++ b/server/models/notification_instance.py @@ -1,35 +1,36 @@ -import datetime -import os -import _io import json import sys import uuid import socket import subprocess -import requests from yattag import indent from json2table import convert # Register NetAlertX directories -INSTALL_PATH="/app" +INSTALL_PATH = "/app" sys.path.extend([f"{INSTALL_PATH}/server"]) -# Register NetAlertX modules +# Register NetAlertX modules import conf -from const import applicationPath, logPath, apiPath, confFileName, reportTemplatesPath -from logger import logResult, mylog -from helper import generate_mac_links, removeDuplicateNewLines, timeNowTZ, get_file_content, write_file, get_setting_value, get_timezone_offset +from const import applicationPath, logPath, apiPath, reportTemplatesPath +from logger import mylog +from helper import generate_mac_links, \ + removeDuplicateNewLines, \ + timeNowTZ, \ + write_file, \ + get_setting_value, \ + get_timezone_offset from messaging.in_app import write_notification -#------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Notification object handling -#------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- class NotificationInstance: def __init__(self, db): self.db = db - # Create Notifications table if missing + # Create Notifications table if missing self.db.sql.execute("""CREATE TABLE IF NOT EXISTS "Notifications" ( "Index" INTEGER, "GUID" TEXT UNIQUE, @@ -48,24 +49,23 @@ class NotificationInstance: self.save() # Method to override processing of notifications - def on_before_create(self, JSON, Extra): + def on_before_create(self, JSON, Extra): return JSON, Extra - # Create a new DB entry if new notifications available, otherwise skip - def create(self, JSON, Extra=""): + def create(self, JSON, Extra=""): JSON, Extra = self.on_before_create(JSON, Extra) # Write output data for debug - write_file (logPath + '/report_output.json', json.dumps(JSON)) - + write_file(logPath + '/report_output.json', json.dumps(JSON)) + # Check if nothing to report, end if JSON["new_devices"] == [] and JSON["down_devices"] == [] and JSON["events"] == [] and JSON["plugins"] == [] and JSON["down_reconnected"] == []: self.HasNotifications = False - else: - self.HasNotifications = True + else: + self.HasNotifications = True self.GUID = str(uuid.uuid4()) self.DateTimeCreated = timeNowTZ() @@ -78,17 +78,14 @@ class NotificationInstance: self.Extra = Extra if self.HasNotifications: - - # if not notiStruc.json['data'] and not notiStruc.text and not notiStruc.html: # mylog('debug', '[Notification] notiStruc is empty') # else: # mylog('debug', ['[Notification] notiStruc:', json.dumps(notiStruc.__dict__, indent=4)]) - - Text = "" - HTML = "" - template_file_path = reportTemplatesPath + 'report_template.html' + Text = "" + HTML = "" + template_file_path = reportTemplatesPath + 'report_template.html' # Open text Template mylog('verbose', ['[Notification] Open text Template']) @@ -99,44 +96,44 @@ class NotificationInstance: # Open html Template mylog('verbose', ['[Notification] Open html Template']) - template_file = open(template_file_path, 'r') + template_file = open(template_file_path, 'r') mail_html = template_file.read() template_file.close() # prepare new version text newVersionText = '' - if conf.newVersionAvailable : + if conf.newVersionAvailable: newVersionText = '🚀A new version is available.' - - mail_text = mail_text.replace ('', newVersionText) - mail_html = mail_html.replace ('', newVersionText) + + mail_text = mail_text.replace('', newVersionText) + mail_html = mail_html.replace('', newVersionText) # Report "REPORT_DATE" in Header & footer - timeFormated = timeNowTZ().strftime ('%Y-%m-%d %H:%M') - mail_text = mail_text.replace ('', timeFormated) - mail_html = mail_html.replace ('', timeFormated) + timeFormated = timeNowTZ().strftime('%Y-%m-%d %H:%M') + mail_text = mail_text.replace('', timeFormated) + mail_html = mail_html.replace('', timeFormated) # Report "SERVER_NAME" in Header & footer - mail_text = mail_text.replace ('', socket.gethostname() ) - mail_html = mail_html.replace ('', socket.gethostname() ) + mail_text = mail_text.replace('', socket.gethostname()) + mail_html = mail_html.replace('', socket.gethostname()) # Report "VERSION" in Header & footer VERSIONFILE = subprocess.check_output(['php', applicationPath + '/front/php/templates/version.php']).decode('utf-8') - mail_text = mail_text.replace ('', VERSIONFILE) - mail_html = mail_html.replace ('', VERSIONFILE) + mail_text = mail_text.replace('', VERSIONFILE) + mail_html = mail_html.replace('', VERSIONFILE) # Report "BUILD" in Header & footer BUILDFILE = subprocess.check_output(['php', applicationPath + '/front/php/templates/build.php']).decode('utf-8') - mail_text = mail_text.replace ('', BUILDFILE) - mail_html = mail_html.replace ('', BUILDFILE) + mail_text = mail_text.replace('', BUILDFILE) + mail_html = mail_html.replace('', BUILDFILE) # Start generating the TEXT & HTML notification messages # new_devices # --- html, text = construct_notifications(self.JSON, "new_devices") - mail_text = mail_text.replace ('', text + '\n') - mail_html = mail_html.replace ('', html) + mail_text = mail_text.replace('', text + '\n') + mail_html = mail_html.replace('', html) mylog('verbose', ['[Notification] New Devices sections done.']) # down_devices @@ -144,56 +141,56 @@ class NotificationInstance: html, text = construct_notifications(self.JSON, "down_devices") - mail_text = mail_text.replace ('', text + '\n') - mail_html = mail_html.replace ('', html) + mail_text = mail_text.replace('', text + '\n') + mail_html = mail_html.replace('', html) mylog('verbose', ['[Notification] Down Devices sections done.']) - + # down_reconnected # --- html, text = construct_notifications(self.JSON, "down_reconnected") - mail_text = mail_text.replace ('', text + '\n') - mail_html = mail_html.replace ('', html) + mail_text = mail_text.replace('', text + '\n') + mail_html = mail_html.replace('', html) mylog('verbose', ['[Notification] Reconnected Down Devices sections done.']) # events # --- - html, text = construct_notifications(self.JSON, "events") - + html, text = construct_notifications(self.JSON, "events") - mail_text = mail_text.replace ('', text + '\n') - mail_html = mail_html.replace ('', html) - mylog('verbose', ['[Notification] Events sections done.']) + + mail_text = mail_text.replace('', text + '\n') + mail_html = mail_html.replace('', html) + mylog('verbose', ['[Notification] Events sections done.']) # plugins # --- html, text = construct_notifications(self.JSON, "plugins") - mail_text = mail_text.replace ('', text + '\n') - mail_html = mail_html.replace ('', html) - + mail_text = mail_text.replace('', text + '\n') + mail_html = mail_html.replace('', html) + mylog('verbose', ['[Notification] Plugins sections done.']) final_text = removeDuplicateNewLines(mail_text) - # Create clickable MAC links - mail_html = generate_mac_links (mail_html, conf.REPORT_DASHBOARD_URL + '/deviceDetails.php?mac=') + # Create clickable MAC links + mail_html = generate_mac_links(mail_html, conf.REPORT_DASHBOARD_URL + '/deviceDetails.php?mac=') final_html = indent( mail_html, - indentation = ' ', - newline = '\r\n', - indent_text = True + indentation=' ', + newline='\r\n', + indent_text=True ) send_api(self.JSON, final_text, final_html) - # Write output data for debug - write_file (logPath + '/report_output.txt', final_text) - write_file (logPath + '/report_output.html', final_html) + # Write output data for debug + write_file(logPath + '/report_output.txt', final_text) + write_file(logPath + '/report_output.html', final_html) mylog('minimal', ['[Notification] Udating API files']) @@ -201,10 +198,10 @@ class NotificationInstance: self.HTML = final_html # Notify frontend - write_notification(f'Report:{self.GUID}', "alert", self.DateTimeCreated ) + write_notification(f'Report:{self.GUID}', "alert", self.DateTimeCreated) self.upsert() - + return self # Only updates the status @@ -216,9 +213,9 @@ class NotificationInstance: def updatePublishedVia(self, newPublishedVia): self.PublishedVia = newPublishedVia self.DateTimePushed = timeNowTZ() - self.upsert() + self.upsert() - # create or update a notification + # create or update a notification def upsert(self): self.db.sql.execute(""" INSERT OR REPLACE INTO Notifications (GUID, DateTimeCreated, DateTimePushed, Status, JSON, Text, HTML, PublishedVia, Extra) @@ -256,57 +253,58 @@ class NotificationInstance: self.save() - - - + # Clear the Pending Email flag from all events and devices def clearPendingEmailFlag(self): # Clean Pending Alert Events - self.db.sql.execute ("""UPDATE Devices SET devLastNotification = ? + self.db.sql.execute("""UPDATE Devices SET devLastNotification = ? WHERE devMac IN ( SELECT eve_MAC FROM Events WHERE eve_PendingAlertEmail = 1 ) - """, (timeNowTZ(),) ) + """, (timeNowTZ(),)) - self.db.sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0 - WHERE eve_PendingAlertEmail = 1 + self.db.sql.execute("""UPDATE Events SET eve_PendingAlertEmail = 0 + WHERE eve_PendingAlertEmail = 1 AND eve_EventType !='Device Down' """) # Clear down events flag after the reporting window passed - self.db.sql.execute (f"""UPDATE Events SET eve_PendingAlertEmail = 0 - WHERE eve_PendingAlertEmail = 1 - AND eve_EventType =='Device Down' + self.db.sql.execute(f"""UPDATE Events SET eve_PendingAlertEmail = 0 + WHERE eve_PendingAlertEmail = 1 + AND eve_EventType =='Device Down' AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}') """) - # clear plugin events - self.db.sql.execute ("DELETE FROM Plugins_Events") + self.db.sql.execute("DELETE FROM Plugins_Events") # DEBUG - print number of rows updated - mylog('minimal', ['[Notification] Notifications changes: ', self.db.sql.rowcount]) + mylog('minimal', ['[Notification] Notifications changes: ', self.db.sql.rowcount]) self.save() + def clearPluginObjects(self): + # clear plugin events + self.db.sql.execute("DELETE FROM Plugins_Events") + self.save() + def save(self): # Commit changes self.db.commitDB() -#------------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- # Reporting -#------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- - - -#------------------------------------------------------------------------------- +# ------------------------------------------------------------------------------ def construct_notifications(JSON, section): - jsn = JSON[section] + jsn = JSON[section] # Return if empty if jsn == []: - return '','' + return '', '' tableTitle = JSON[section + "_meta"]["title"] headers = JSON[section + "_meta"]["columnNames"] @@ -314,22 +312,34 @@ def construct_notifications(JSON, section): html = '' text = '' - table_attributes = {"style" : "border-collapse: collapse; font-size: 12px; color:#70707", "width" : "100%", "cellspacing" : 0, "cellpadding" : "3px", "bordercolor" : "#C0C0C0", "border":"1"} + table_attributes = { + "style": "border-collapse: collapse; font-size: 12px; color:#70707", + "width": "100%", + "cellspacing": 0, + "cellpadding": "3px", + "bordercolor": "#C0C0C0", + "border": "1" + } headerProps = "width='120px' style='color:white; font-size: 16px;' bgcolor='#64a0d6' " thProps = "width='120px' style='color:#F0F0F0' bgcolor='#64a0d6' " build_direction = "TOP_TO_BOTTOM" text_line = '{}\t{}\n' - if len(jsn) > 0: text = tableTitle + "\n---------\n" # Convert a JSON into an HTML table html = convert({"data": jsn}, build_direction=build_direction, table_attributes=table_attributes) - + # Cleanup the generated HTML table notification - html = format_table(html, "data", headerProps, tableTitle).replace('
    ','
      ').replace("null", "") + html = format_table(html, + "data", + headerProps, + tableTitle).replace('
        ', + '
          ' + ).replace("null", + "") # prepare text-only message for device in jsn: @@ -337,7 +347,7 @@ def construct_notifications(JSON, section): padding = "" if len(header) < 4: padding = "\t" - text += text_line.format ( header + ': ' + padding, device[header]) + text += text_line.format(header + ': ' + padding, device[header]) text += '\n' # Format HTML table headers @@ -346,24 +356,21 @@ def construct_notifications(JSON, section): return html, text -#------------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------- def send_api(json_final, mail_text, mail_html): - mylog('verbose', ['[Send API] Updating notification_* files in ', apiPath]) + mylog('verbose', ['[Send API] Updating notification_* files in ', apiPath]) - write_file(apiPath + 'notification_text.txt' , mail_text) - write_file(apiPath + 'notification_text.html' , mail_html) - write_file(apiPath + 'notification_json_final.json' , json.dumps(json_final)) + write_file(apiPath + 'notification_text.txt', mail_text) + write_file(apiPath + 'notification_text.html', mail_html) + write_file(apiPath + 'notification_json_final.json', json.dumps(json_final)) - -#------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- # Replacing table headers -def format_table (html, thValue, props, newThValue = ''): +def format_table(html, thValue, props, newThValue=''): if newThValue == '': newThValue = thValue - return html.replace(""+thValue+"", ""+newThValue+"" ) - - - + return html.replace(""+thValue+"", ""+newThValue+"") From 8cbfd04db6aeaf3b0d6ed9be08eb1fa8452e7144 Mon Sep 17 00:00:00 2001 From: Ingo Ratsdorf Date: Tue, 16 Sep 2025 07:49:17 +1200 Subject: [PATCH 2/3] Renamed sub for readability --- server/__main__.py | 2 +- server/models/notification_instance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/__main__.py b/server/__main__.py index a8369299..3412dfad 100755 --- a/server/__main__.py +++ b/server/__main__.py @@ -189,7 +189,7 @@ def main (): notification.clearPendingEmailFlag() else: - notification.clearPluginObjects() + notification.clearPluginEvents() mylog('verbose', ['[Notification] No changes to report']) # Commit SQL diff --git a/server/models/notification_instance.py b/server/models/notification_instance.py index 403cbd8e..a2cfbef2 100755 --- a/server/models/notification_instance.py +++ b/server/models/notification_instance.py @@ -283,7 +283,7 @@ class NotificationInstance: self.save() - def clearPluginObjects(self): + def clearPluginEvents(self): # clear plugin events self.db.sql.execute("DELETE FROM Plugins_Events") self.save() From a51d0e72c7cbd46407d174f7541756c7f85d5a68 Mon Sep 17 00:00:00 2001 From: Ingo Ratsdorf Date: Wed, 17 Sep 2025 08:58:02 +1200 Subject: [PATCH 3/3] DRY fix avoiding repeat code in notification_instance. Still a refactor would be great as the plugins_events table is getting filled in plugin.py and thus should be cleared in there. --- server/__main__.py | 5 ++ server/models/notification_instance.py | 65 +++++++++++++++++--------- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/server/__main__.py b/server/__main__.py index 3412dfad..591a3c4e 100755 --- a/server/__main__.py +++ b/server/__main__.py @@ -186,9 +186,14 @@ def main (): pm.run_plugin_scripts('on_notification') notification.setAllProcessed() + + # clear pending email flag + # and the plugin events notification.clearPendingEmailFlag() else: + # If there are no notifications to process, + # we still need to clear all plugin events notification.clearPluginEvents() mylog('verbose', ['[Notification] No changes to report']) diff --git a/server/models/notification_instance.py b/server/models/notification_instance.py index a2cfbef2..dabad488 100755 --- a/server/models/notification_instance.py +++ b/server/models/notification_instance.py @@ -118,12 +118,28 @@ class NotificationInstance: mail_html = mail_html.replace('', socket.gethostname()) # Report "VERSION" in Header & footer - VERSIONFILE = subprocess.check_output(['php', applicationPath + '/front/php/templates/version.php']).decode('utf-8') + try: + VERSIONFILE = subprocess.check_output( + ['php', applicationPath + '/front/php/templates/version.php'], + timeout=5 + ).decode('utf-8') + except Exception as e: + mylog('debug', [f'[Notification] Unable to read version.php: {e}']) + VERSIONFILE = 'unknown' + mail_text = mail_text.replace('', VERSIONFILE) mail_html = mail_html.replace('', VERSIONFILE) # Report "BUILD" in Header & footer - BUILDFILE = subprocess.check_output(['php', applicationPath + '/front/php/templates/build.php']).decode('utf-8') + try: + BUILDFILE = subprocess.check_output( + ['php', applicationPath + '/front/php/templates/build.php'], + timeout=5 + ).decode('utf-8') + except Exception as e: + mylog('debug', [f'[Notification] Unable to read build.php: {e}']) + BUILDFILE = 'unknown' + mail_text = mail_text.replace('', BUILDFILE) mail_html = mail_html.replace('', BUILDFILE) @@ -257,34 +273,39 @@ class NotificationInstance: def clearPendingEmailFlag(self): # Clean Pending Alert Events - self.db.sql.execute("""UPDATE Devices SET devLastNotification = ? - WHERE devMac IN ( - SELECT eve_MAC FROM Events - WHERE eve_PendingAlertEmail = 1 - ) - """, (timeNowTZ(),)) + self.db.sql.execute(""" + UPDATE Devices SET devLastNotification = ? + WHERE devMac IN ( + SELECT eve_MAC FROM Events + WHERE eve_PendingAlertEmail = 1 + ) + """, (timeNowTZ(),)) - self.db.sql.execute("""UPDATE Events SET eve_PendingAlertEmail = 0 - WHERE eve_PendingAlertEmail = 1 - AND eve_EventType !='Device Down' """) + self.db.sql.execute(""" + UPDATE Events SET eve_PendingAlertEmail = 0 + WHERE eve_PendingAlertEmail = 1 + AND eve_EventType !='Device Down' """) # Clear down events flag after the reporting window passed - self.db.sql.execute(f"""UPDATE Events SET eve_PendingAlertEmail = 0 - WHERE eve_PendingAlertEmail = 1 - AND eve_EventType =='Device Down' - AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}') - """) + minutes = int(get_setting_value('NTFPRCS_alert_down_time') or 0) + tz_offset = get_timezone_offset() + self.db.sql.execute(""" + UPDATE Events + SET eve_PendingAlertEmail = 0 + WHERE eve_PendingAlertEmail = 1 + AND eve_EventType = 'Device Down' + AND eve_DateTime < datetime('now', ?, ?) + """, (f"-{minutes} minutes", tz_offset)) + + mylog('minimal', ['[Notification] Notifications changes: ', + self.db.sql.rowcount]) # clear plugin events - self.db.sql.execute("DELETE FROM Plugins_Events") + self.clearPluginEvents() - # DEBUG - print number of rows updated - mylog('minimal', ['[Notification] Notifications changes: ', self.db.sql.rowcount]) - - self.save() def clearPluginEvents(self): - # clear plugin events + # clear plugin events table self.db.sql.execute("DELETE FROM Plugins_Events") self.save()