Notifications refactor and #242 📩

front/devices.php

@@ -327,7 +327,7 @@ function filterDataByStatus(data, status) {
       case 'new':
         return item.dev_NewDevice === 1;
       case 'down':
-        return item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown  === 1;
+        return item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown  !== 0;
       case 'archived':
         return item.dev_Archived === 1;
       default:
@@ -343,7 +343,7 @@ function getDeviceStatus(item)
     {
       return 'On-line';
     }
-    else if(item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown  === 1)
+    else if(item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown  !== 0)
     {
       return 'Down';
     }

front/php/server/devices.php

@@ -77,7 +77,7 @@ function getDeviceData() {
 
   // Device Data
   $sql = 'SELECT rowid, *,
-            CASE WHEN dev_AlertDeviceDown=1 AND dev_PresentLastScan=0 THEN "Down"
+            CASE WHEN dev_AlertDeviceDown !=0 AND dev_PresentLastScan=0 THEN "Down"
                  WHEN dev_PresentLastScan=1 THEN "On-line"
                  ELSE "Off-line" END as dev_Status
           FROM Devices
@@ -626,7 +626,7 @@ function getDevicesList() {
 
   $sql = 'SELECT * FROM (
               SELECT rowid, *, CASE
-                      WHEN t1.dev_AlertDeviceDown=1 AND t1.dev_PresentLastScan=0 THEN "Down"
+                      WHEN t1.dev_AlertDeviceDown !=0 AND t1.dev_PresentLastScan=0 THEN "Down"
                       WHEN t1.dev_NewDevice=1 THEN "New"
                       WHEN t1.dev_PresentLastScan=1 THEN "On-line"
                       ELSE "Off-line"  END AS dev_Status
@@ -1133,14 +1133,14 @@ function copyFromDevice() {
 //------------------------------------------------------------------------------
 function getDeviceCondition ($deviceStatus) {
   switch ($deviceStatus) {
-    case 'all':        return 'WHERE dev_Archived=0';                                                      break;
+    case 'all':        return 'WHERE dev_Archived=0';                                                        break;
-    case 'connected':  return 'WHERE dev_Archived=0 AND dev_PresentLastScan=1';                            break;
+    case 'connected':  return 'WHERE dev_Archived=0 AND dev_PresentLastScan=1';                              break;
-    case 'favorites':  return 'WHERE dev_Archived=0 AND dev_Favorite=1';                                   break;
+    case 'favorites':  return 'WHERE dev_Archived=0 AND dev_Favorite=1';                                     break;
-    case 'new':        return 'WHERE dev_Archived=0 AND dev_NewDevice=1';                                  break;
+    case 'new':        return 'WHERE dev_Archived=0 AND dev_NewDevice=1';                                    break;
-    case 'down':       return 'WHERE dev_Archived=0 AND dev_AlertDeviceDown=1 AND dev_PresentLastScan=0';  break;
+    case 'down':       return 'WHERE dev_Archived=0 AND dev_AlertDeviceDown !=0 AND dev_PresentLastScan=0';  break;
-    case 'archived':   return 'WHERE dev_Archived=1';                                                      break;
+    case 'archived':   return 'WHERE dev_Archived=1';                                                        break;
-    default:           return 'WHERE 1=0';                                                                 break;
+    default:           return 'WHERE 1=0';                                                                   break;
-  }
+  }  
 }
 
 

front/plugins/notification_processing/README.md

@@ -0,0 +1,7 @@
+## Overview
+
+Plugin to run regular database cleanup tasks. It is strongly recommended to have an hourly or at least daily schedule running.
+
+### Usage
+
+- Check the Settings page for details.

front/plugins/notification_processing/config.json

@@ -0,0 +1,150 @@
+{
+  "code_name": "notification_processing",
+  "unique_prefix": "NTFPRCS",
+  "plugin_type": "system",
+  "enabled": true,
+  "data_source": "script",  
+  "show_ui": false,
+  "localized": ["display_name", "description", "icon"],
+  "display_name": [
+    {
+      "language_code": "en_us",
+      "string": "Notification Processing"
+    }
+  ],
+  "icon": [
+    {
+      "language_code": "en_us",
+      "string": "<i class=\"fa-solid fa-envelopes-bulk\"></i>"
+    }
+  ],
+  "description": [
+    {
+      "language_code": "en_us",
+      "string": "A plugin to for advanced notification processing."
+    }  
+  ],
+    "params" : [
+  ],  
+
+  "settings": [
+    {
+      "function": "RUN",   
+      "events": ["run"],         
+      "type": "text.select",            
+      "default_value":"schedule",
+      "options": ["disabled", "before_notification"],
+      "localized": ["name", "description"],
+      "name" :[{
+          "language_code":"en_us",
+          "string" : "When to run"
+      },
+      {
+          "language_code":"es_es",
+          "string" : "Cuándo ejecutar"
+      },
+      {
+          "language_code":"de_de",
+          "string" : "Wann laufen"
+      }],
+      "description": [{
+          "language_code":"en_us",
+          "string" : "When the Notification manipulation should happen. Usually set to <code>before_notification</code>."
+      }]
+  },    
+    {
+      "function": "CMD",
+      "type": "readonly",
+      "default_value": "python3 /home/pi/pialert/front/plugins/notification_processing/script.py",
+      "options": [],
+      "localized": ["name", "description"],
+      "name": [
+        {
+          "language_code": "en_us",
+          "string": "Command"
+        },
+        {
+          "language_code": "es_es",
+          "string": "Comando"
+        },
+        {
+          "language_code": "de_de",
+          "string": "Befehl"
+        }	        
+      ],
+      "description": [
+        {
+          "language_code": "en_us",
+          "string": "Command to run. This can not be changed"
+        },
+        {
+          "language_code": "es_es",
+          "string": "Comando a ejecutar. Esto no se puede cambiar"
+        },
+        {
+          "language_code": "de_de",
+          "string": "Befehl zum Ausführen. Dies kann nicht geändert werden"
+        }	        
+      ]
+    },    
+    {
+      "function": "RUN_TIMEOUT",
+      "type": "integer",
+      "default_value": 30,
+      "options": [],
+      "localized": ["name", "description"],
+      "name": [
+        {
+          "language_code": "en_us",
+          "string": "Run timeout"
+        },
+        {
+          "language_code": "es_es",
+          "string": "Tiempo límite de ejecución"
+        },
+        {
+          "language_code": "de_de",
+          "string": "Zeitüberschreitung"
+        }	        
+      ],
+      "description": [
+        {
+          "language_code": "en_us",
+          "string": "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted."
+        },
+        {
+          "language_code": "es_es",
+          "string": "Tiempo máximo en segundos para esperar a que finalice el script. Si se supera este tiempo, el script se cancela."
+        },
+        {
+          "language_code": "de_de",
+          "string": "Maximale Zeit in Sekunden, die auf den Abschluss des Skripts gewartet werden soll. Bei Überschreitung dieser Zeit wird das Skript abgebrochen."
+        }        
+      ]
+    },
+    {
+      "function": "alert_down_time",
+      "type": "integer",
+      "default_value": 8,
+      "options": [],
+      "localized": ["name", "description"],
+      "name": [
+        {
+          "language_code": "en_us",
+          "string": "Alert Down After"
+        }
+      ],
+      "description": [
+        {
+          "language_code": "en_us",
+          "string": "After how many minutes a down device is reported."
+        }    
+      ]    
+    }
+  ],
+
+  "database_column_definitions": 
+  [          
+           
+  ]
+}

front/plugins/notification_processing/script.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+import os
+import pathlib
+import argparse
+import sys
+import hashlib
+import csv
+import sqlite3
+from io import StringIO
+from datetime import datetime
+
+sys.path.append("/home/pi/pialert/front/plugins")
+sys.path.append('/home/pi/pialert/pialert') 
+
+from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64
+from logger import mylog, append_line_to_file
+from helper import timeNowTZ, get_setting_value
+from const import logPath, pialertPath
+
+
+CUR_PATH = str(pathlib.Path(__file__).parent.resolve())
+LOG_FILE = os.path.join(CUR_PATH, 'script.log')
+RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
+
+pluginName= 'NTFPRCS'
+
+def main():
+
+    mylog('verbose', [f'[{pluginName}] In script'])
+    
+    # TODO
+    # process_notifications('/home/pi/pialert/db/pialert.db')
+    
+    mylog('verbose', [f'[{pluginName}] Script finished'])
+    
+    return 0
+
+#===============================================================================
+# Cleanup / upkeep database
+#===============================================================================
+def process_notifications (dbPath):
+    """
+    Cleaning out old records from the tables that don't need to keep all data.
+    """
+    # Connect to the PiAlert SQLite database
+    conn    = sqlite3.connect(dbPath)
+    cursor  = conn.cursor()
+
+    # Cleanup Events
+    # mylog('verbose', [f'[DBCLNP] Events: Delete all older than {str(DAYS_TO_KEEP_EVENTS)} days (DAYS_TO_KEEP_EVENTS setting)'])
+
+    # cursor.execute (f"""DELETE FROM Events 
+    #                         WHERE eve_DateTime <= date('now', '-{str(DAYS_TO_KEEP_EVENTS)} day')""")
+               
+
+    conn.commit()
+
+    # Close the database connection
+    conn.close()
+    
+    
+
+#===============================================================================
+# BEGIN
+#===============================================================================
+if __name__ == '__main__':
+    main()

pialert/__main__.py

@@ -161,22 +161,9 @@ def main ():
             if notificationObj.HasNotifications:
                 pluginsState = run_plugin_scripts(db, 'on_notification', pluginsState) 
                 notification.setAllProcessed()
+                notification.clearPendingEmailFlag()
 
-                # Clean Pending Alert Events
+                
-                sql.execute ("""UPDATE Devices SET dev_LastNotification = ?
-                                    WHERE dev_MAC IN (
-                                        SELECT eve_MAC FROM Events
-                                            WHERE eve_PendingAlertEmail = 1
-                                    )
-                             """, (timeNowTZ(),) )
-                sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
-                                    WHERE eve_PendingAlertEmail = 1""")
-
-                # clear plugin events
-                sql.execute ("DELETE FROM Plugins_Events")
-
-                # DEBUG - print number of rows updated
-                mylog('minimal', ['[Notification] Notifications changes: ', sql.rowcount])
             else:
                 mylog('verbose', ['[Notification] No changes to report'])
 

pialert/device.py

@@ -41,8 +41,8 @@ def print_scan_stats(db):
     SELECT
         (SELECT COUNT(*) FROM CurrentScan) AS devices_detected,
         (SELECT COUNT(*) FROM CurrentScan WHERE NOT EXISTS (SELECT 1 FROM Devices WHERE dev_MAC = cur_MAC)) AS new_devices,
-        (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS down_alerts,
+        (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown != 0 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS down_alerts,
-        (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown = 1 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS new_down_alerts,
+        (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown != 0 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS new_down_alerts,
         (SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 0) AS new_connections,
         (SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS disconnections,
         (SELECT COUNT(*) FROM Devices, CurrentScan WHERE dev_MAC = cur_MAC AND dev_LastIP <> cur_IP) AS ip_changes,

pialert/helper.py

@@ -37,6 +37,12 @@ def timeNowTZ():
 def timeNow():
     return datetime.datetime.now().replace(microsecond=0)
 
+def get_timezone_offset():    
+    now = datetime.datetime.now(conf.tz)
+    offset_hours = now.utcoffset().total_seconds() / 3600        
+    offset_formatted =  "{:+03d}:{:02d}".format(int(offset_hours), int((offset_hours % 1) * 60))
+    return offset_formatted
+
 
 #-------------------------------------------------------------------------------
 # App state
@@ -470,58 +476,6 @@ def resolve_device_name_pholus (pMAC, pIP, allRes, nameNotFound, match_IP = Fals
         if 'PTR Class:IN' in value and len(value.split('"')) > 1:
             return cleanDeviceName(value.split('"')[1], match_IP)
 
-
-    # # airplay matches contain a lot of information
-    # # Matches for example: 
-    # # Brand Tv (50)._airplay._tcp.local. TXT Class:32769 "acl=0 deviceid=66:66:66:66:66:66 features=0x77777,0x38BCB46 rsf=0x3 fv=p20.T-FFFFFF-03.1 flags=0x204 model=XXXX manufacturer=Brand serialNumber=XXXXXXXXXXX protovers=1.1 srcvers=777.77.77 pi=FF:FF:FF:FF:FF:FF psi=00000000-0000-0000-0000-FFFFFFFFFF gid=00000000-0000-0000-0000-FFFFFFFFFF gcgl=0 pk=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
-    # for i in pholusMatchesIndexes:
-    #     if checkIPV4(allRes[i]['IP_v4_or_v6']) and '._airplay._tcp.local. TXT Class:32769' in str(allRes[i]["Value"]) :
-    #         return cleanDeviceName(allRes[i]["Value"].split('._airplay._tcp.local. TXT Class:32769')[0], match_IP)
-    
-    # # second best - contains airplay
-    # # Matches for example: 
-    # # _airplay._tcp.local. PTR Class:IN "Brand Tv (50)._airplay._tcp.local."
-    # for i in pholusMatchesIndexes:
-    #     if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_airplay._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('._googlecast') not in allRes[i]["Value"]:
-    #         return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP)    
-
-    # # Contains PTR Class:32769
-    # # Matches for example: 
-    # # 3.1.168.192.in-addr.arpa. PTR Class:32769 "MyPc.local."
-    # for i in pholusMatchesIndexes:
-    #     if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:32769' in allRes[i]["Value"]:
-    #         return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP)
-
-    # # Contains AAAA Class:IN
-    # # Matches for example: 
-    # # DESKTOP-SOMEID.local. AAAA Class:IN "fe80::fe80:fe80:fe80:fe80"
-    # for i in pholusMatchesIndexes:
-    #     if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'AAAA Class:IN' in allRes[i]["Value"]:
-    #         return cleanDeviceName(allRes[i]["Value"].split('.local.')[0], match_IP)
-
-    # # Contains _googlecast._tcp.local. PTR Class:IN
-    # # Matches for example: 
-    # # _googlecast._tcp.local. PTR Class:IN "Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77._googlecast._tcp.local."
-    # for i in pholusMatchesIndexes:
-    #     if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_googlecast._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('Google-Cast-Group') not in allRes[i]["Value"]:
-    #         return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP)
-
-    # # Contains A Class:32769
-    # # Matches for example: 
-    # # Android.local. A Class:32769 "192.168.1.6"
-    # for i in pholusMatchesIndexes:
-    #     if checkIPV4(allRes[i]['IP_v4_or_v6']) and ' A Class:32769' in allRes[i]["Value"]:
-    #         return cleanDeviceName(allRes[i]["Value"].split(' A Class:32769')[0], match_IP)
-
-
-    # # # Contains PTR Class:IN
-    # # Matches for example: 
-    # # _esphomelib._tcp.local. PTR Class:IN "ceiling-light-1._esphomelib._tcp.local."
-    # for i in pholusMatchesIndexes:
-    #     if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:IN' in allRes[i]["Value"]:
-    #         if allRes[i]["Value"] and len(allRes[i]["Value"].split('"')) > 1:
-    #             return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP)
-
     return nameNotFound
     
     

pialert/networkscan.py

@@ -187,7 +187,7 @@ def insert_events (db):
                         eve_PendingAlertEmail)
                     SELECT dev_MAC, dev_LastIP, '{startTime}', 'Device Down', '', 1
                     FROM Devices 
-                    WHERE dev_AlertDeviceDown = 1
+                    WHERE dev_AlertDeviceDown != 0
                       AND dev_PresentLastScan = 1                      
                       AND NOT EXISTS (SELECT 1 FROM CurrentScan
                                       WHERE dev_MAC = cur_MAC

pialert/notification.py

@@ -10,7 +10,7 @@ import conf
 import const
 from const import pialertPath, logPath, apiPath
 from logger import logResult, mylog, print_log
-from helper import generate_mac_links, removeDuplicateNewLines, timeNowTZ, get_file_content, write_file
+from helper import generate_mac_links, removeDuplicateNewLines, timeNowTZ, get_file_content, write_file, get_setting_value, get_timezone_offset
 
 #-------------------------------------------------------------------------------
 # Notification object handling
@@ -209,7 +209,34 @@ class Notification_obj:
 
         self.save()
 
-        
+    def clearPendingEmailFlag(self):
+
+        # Clean Pending Alert Events
+        self.db.sql.execute ("""UPDATE Devices SET dev_LastNotification = ?
+                            WHERE dev_MAC IN (
+                                SELECT eve_MAC FROM Events
+                                    WHERE eve_PendingAlertEmail = 1
+                            )
+                        """, (timeNowTZ(),) )
+
+        self.db.sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
+                                    WHERE eve_PendingAlertEmail = 1 
+                                    AND eve_EventType !='Device Down' """)
+
+        # Clear down events flag after the reporting window passed
+        self.db.sql.execute (f"""UPDATE Events SET eve_PendingAlertEmail = 0
+                                    WHERE eve_PendingAlertEmail = 1 
+                                    AND eve_EventType =='Device Down' 
+                                    AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}')
+                            """)
+
+        # clear plugin events
+        self.db.sql.execute ("DELETE FROM Plugins_Events")
+
+        # DEBUG - print number of rows updated
+        mylog('minimal', ['[Notification] Notifications changes: ', self.db.sql.rowcount])    
+
+        self.save()
 
     def save(self):
         # Commit changes

pialert/reporting.py

@@ -17,7 +17,7 @@ import json
 import conf
 import const
 from const import pialertPath, logPath, apiPath
-from helper import timeNowTZ, get_file_content, write_file
+from helper import timeNowTZ, get_file_content, write_file, get_timezone_offset, get_setting_value
 from logger import logResult, mylog, print_log
 
 
@@ -74,10 +74,21 @@ def get_notifications (db):
 
     if 'down_devices' in conf.INCLUDED_SECTIONS :
         # Compose Devices Down Section
-        sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments  FROM Events_Devices
+        sqlQuery = f"""
-                        WHERE eve_PendingAlertEmail = 1
+                    SELECT *
-                        AND eve_EventType = 'Device Down'
+                        FROM Events AS down_events
-                        ORDER BY eve_DateTime"""
+                        WHERE eve_PendingAlertEmail = 1 
+                        AND down_events.eve_EventType = 'Device Down' 
+                        AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}')
+                        AND NOT EXISTS (
+                            SELECT 1
+                            FROM Events AS connected_events
+                            WHERE connected_events.eve_MAC = down_events.eve_MAC
+                                AND connected_events.eve_EventType = 'Connected'
+                                AND connected_events.eve_DateTime > down_events.eve_DateTime        
+                        )
+                        ORDER BY down_events.eve_DateTime;
+                    """
         
         # Get the events as JSON        
         json_obj = db.get_table_as_json(sqlQuery)
@@ -139,7 +150,8 @@ def skip_repeated_notifications (db):
 
     # Skip repeated notifications
     # due strfime : Overflow --> use  "strftime / 60"
-    mylog('verbose','[Skip Repeated Notifications] Skip Repeated start')
+    mylog('verbose','[Skip Repeated Notifications] Skip Repeated')
+    
     db.sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0
                     WHERE eve_PendingAlertEmail = 1 AND eve_MAC IN
                         (
@@ -151,7 +163,7 @@ def skip_repeated_notifications (db):
                               (strftime('%s','now','localtime')/60 )
                         )
                  """ )
-    mylog('verbose','[Skip Repeated Notifications] Skip Repeated end')
+   
 
     db.commitDB()