diff --git a/front/devices.php b/front/devices.php index be7c8ab4..f430d830 100755 --- a/front/devices.php +++ b/front/devices.php @@ -327,7 +327,7 @@ function filterDataByStatus(data, status) { case 'new': return item.dev_NewDevice === 1; case 'down': - return item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown === 1; + return item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown !== 0; case 'archived': return item.dev_Archived === 1; default: @@ -343,7 +343,7 @@ function getDeviceStatus(item) { return 'On-line'; } - else if(item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown === 1) + else if(item.dev_PresentLastScan === 0 && item.dev_AlertDeviceDown !== 0) { return 'Down'; } diff --git a/front/php/server/devices.php b/front/php/server/devices.php index 8a87c0f0..698c0cbb 100755 --- a/front/php/server/devices.php +++ b/front/php/server/devices.php @@ -77,7 +77,7 @@ function getDeviceData() { // Device Data $sql = 'SELECT rowid, *, - CASE WHEN dev_AlertDeviceDown=1 AND dev_PresentLastScan=0 THEN "Down" + CASE WHEN dev_AlertDeviceDown !=0 AND dev_PresentLastScan=0 THEN "Down" WHEN dev_PresentLastScan=1 THEN "On-line" ELSE "Off-line" END as dev_Status FROM Devices @@ -626,7 +626,7 @@ function getDevicesList() { $sql = 'SELECT * FROM ( SELECT rowid, *, CASE - WHEN t1.dev_AlertDeviceDown=1 AND t1.dev_PresentLastScan=0 THEN "Down" + WHEN t1.dev_AlertDeviceDown !=0 AND t1.dev_PresentLastScan=0 THEN "Down" WHEN t1.dev_NewDevice=1 THEN "New" WHEN t1.dev_PresentLastScan=1 THEN "On-line" ELSE "Off-line" END AS dev_Status @@ -1133,14 +1133,14 @@ function copyFromDevice() { //------------------------------------------------------------------------------ function getDeviceCondition ($deviceStatus) { switch ($deviceStatus) { - case 'all': return 'WHERE dev_Archived=0'; break; - case 'connected': return 'WHERE dev_Archived=0 AND dev_PresentLastScan=1'; break; - case 'favorites': return 'WHERE dev_Archived=0 AND dev_Favorite=1'; break; - case 'new': return 'WHERE dev_Archived=0 AND dev_NewDevice=1'; break; - case 'down': return 'WHERE dev_Archived=0 AND dev_AlertDeviceDown=1 AND dev_PresentLastScan=0'; break; - case 'archived': return 'WHERE dev_Archived=1'; break; - default: return 'WHERE 1=0'; break; - } + case 'all': return 'WHERE dev_Archived=0'; break; + case 'connected': return 'WHERE dev_Archived=0 AND dev_PresentLastScan=1'; break; + case 'favorites': return 'WHERE dev_Archived=0 AND dev_Favorite=1'; break; + case 'new': return 'WHERE dev_Archived=0 AND dev_NewDevice=1'; break; + case 'down': return 'WHERE dev_Archived=0 AND dev_AlertDeviceDown !=0 AND dev_PresentLastScan=0'; break; + case 'archived': return 'WHERE dev_Archived=1'; break; + default: return 'WHERE 1=0'; break; + } } diff --git a/front/plugins/notification_processing/README.md b/front/plugins/notification_processing/README.md new file mode 100755 index 00000000..03fa70f4 --- /dev/null +++ b/front/plugins/notification_processing/README.md @@ -0,0 +1,7 @@ +## Overview + +Plugin to run regular database cleanup tasks. It is strongly recommended to have an hourly or at least daily schedule running. + +### Usage + +- Check the Settings page for details. diff --git a/front/plugins/notification_processing/config.json b/front/plugins/notification_processing/config.json new file mode 100755 index 00000000..42a47638 --- /dev/null +++ b/front/plugins/notification_processing/config.json @@ -0,0 +1,150 @@ +{ + "code_name": "notification_processing", + "unique_prefix": "NTFPRCS", + "plugin_type": "system", + "enabled": true, + "data_source": "script", + "show_ui": false, + "localized": ["display_name", "description", "icon"], + "display_name": [ + { + "language_code": "en_us", + "string": "Notification Processing" + } + ], + "icon": [ + { + "language_code": "en_us", + "string": "" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "A plugin to for advanced notification processing." + } + ], + "params" : [ + ], + + "settings": [ + { + "function": "RUN", + "events": ["run"], + "type": "text.select", + "default_value":"schedule", + "options": ["disabled", "before_notification"], + "localized": ["name", "description"], + "name" :[{ + "language_code":"en_us", + "string" : "When to run" + }, + { + "language_code":"es_es", + "string" : "Cuándo ejecutar" + }, + { + "language_code":"de_de", + "string" : "Wann laufen" + }], + "description": [{ + "language_code":"en_us", + "string" : "When the Notification manipulation should happen. Usually set to before_notification." + }] + }, + { + "function": "CMD", + "type": "readonly", + "default_value": "python3 /home/pi/pialert/front/plugins/notification_processing/script.py", + "options": [], + "localized": ["name", "description"], + "name": [ + { + "language_code": "en_us", + "string": "Command" + }, + { + "language_code": "es_es", + "string": "Comando" + }, + { + "language_code": "de_de", + "string": "Befehl" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "Command to run. This can not be changed" + }, + { + "language_code": "es_es", + "string": "Comando a ejecutar. Esto no se puede cambiar" + }, + { + "language_code": "de_de", + "string": "Befehl zum Ausführen. Dies kann nicht geändert werden" + } + ] + }, + { + "function": "RUN_TIMEOUT", + "type": "integer", + "default_value": 30, + "options": [], + "localized": ["name", "description"], + "name": [ + { + "language_code": "en_us", + "string": "Run timeout" + }, + { + "language_code": "es_es", + "string": "Tiempo límite de ejecución" + }, + { + "language_code": "de_de", + "string": "Zeitüberschreitung" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted." + }, + { + "language_code": "es_es", + "string": "Tiempo máximo en segundos para esperar a que finalice el script. Si se supera este tiempo, el script se cancela." + }, + { + "language_code": "de_de", + "string": "Maximale Zeit in Sekunden, die auf den Abschluss des Skripts gewartet werden soll. Bei Überschreitung dieser Zeit wird das Skript abgebrochen." + } + ] + }, + { + "function": "alert_down_time", + "type": "integer", + "default_value": 8, + "options": [], + "localized": ["name", "description"], + "name": [ + { + "language_code": "en_us", + "string": "Alert Down After" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "After how many minutes a down device is reported." + } + ] + } + ], + + "database_column_definitions": + [ + + ] +} diff --git a/front/plugins/notification_processing/script.py b/front/plugins/notification_processing/script.py new file mode 100755 index 00000000..0af578ff --- /dev/null +++ b/front/plugins/notification_processing/script.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +import os +import pathlib +import argparse +import sys +import hashlib +import csv +import sqlite3 +from io import StringIO +from datetime import datetime + +sys.path.append("/home/pi/pialert/front/plugins") +sys.path.append('/home/pi/pialert/pialert') + +from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64 +from logger import mylog, append_line_to_file +from helper import timeNowTZ, get_setting_value +from const import logPath, pialertPath + + +CUR_PATH = str(pathlib.Path(__file__).parent.resolve()) +LOG_FILE = os.path.join(CUR_PATH, 'script.log') +RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log') + +pluginName= 'NTFPRCS' + +def main(): + + mylog('verbose', [f'[{pluginName}] In script']) + + # TODO + # process_notifications('/home/pi/pialert/db/pialert.db') + + mylog('verbose', [f'[{pluginName}] Script finished']) + + return 0 + +#=============================================================================== +# Cleanup / upkeep database +#=============================================================================== +def process_notifications (dbPath): + """ + Cleaning out old records from the tables that don't need to keep all data. + """ + # Connect to the PiAlert SQLite database + conn = sqlite3.connect(dbPath) + cursor = conn.cursor() + + # Cleanup Events + # mylog('verbose', [f'[DBCLNP] Events: Delete all older than {str(DAYS_TO_KEEP_EVENTS)} days (DAYS_TO_KEEP_EVENTS setting)']) + + # cursor.execute (f"""DELETE FROM Events + # WHERE eve_DateTime <= date('now', '-{str(DAYS_TO_KEEP_EVENTS)} day')""") + + + conn.commit() + + # Close the database connection + conn.close() + + + +#=============================================================================== +# BEGIN +#=============================================================================== +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/pialert/__main__.py b/pialert/__main__.py index fc4be05f..16c204c3 100755 --- a/pialert/__main__.py +++ b/pialert/__main__.py @@ -161,22 +161,9 @@ def main (): if notificationObj.HasNotifications: pluginsState = run_plugin_scripts(db, 'on_notification', pluginsState) notification.setAllProcessed() + notification.clearPendingEmailFlag() - # Clean Pending Alert Events - sql.execute ("""UPDATE Devices SET dev_LastNotification = ? - WHERE dev_MAC IN ( - SELECT eve_MAC FROM Events - WHERE eve_PendingAlertEmail = 1 - ) - """, (timeNowTZ(),) ) - sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0 - WHERE eve_PendingAlertEmail = 1""") - - # clear plugin events - sql.execute ("DELETE FROM Plugins_Events") - - # DEBUG - print number of rows updated - mylog('minimal', ['[Notification] Notifications changes: ', sql.rowcount]) + else: mylog('verbose', ['[Notification] No changes to report']) diff --git a/pialert/device.py b/pialert/device.py index eda19ad1..f4541058 100755 --- a/pialert/device.py +++ b/pialert/device.py @@ -41,8 +41,8 @@ def print_scan_stats(db): SELECT (SELECT COUNT(*) FROM CurrentScan) AS devices_detected, (SELECT COUNT(*) FROM CurrentScan WHERE NOT EXISTS (SELECT 1 FROM Devices WHERE dev_MAC = cur_MAC)) AS new_devices, - (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS down_alerts, - (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown = 1 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS new_down_alerts, + (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown != 0 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS down_alerts, + (SELECT COUNT(*) FROM Devices WHERE dev_AlertDeviceDown != 0 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS new_down_alerts, (SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 0) AS new_connections, (SELECT COUNT(*) FROM Devices WHERE dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC)) AS disconnections, (SELECT COUNT(*) FROM Devices, CurrentScan WHERE dev_MAC = cur_MAC AND dev_LastIP <> cur_IP) AS ip_changes, diff --git a/pialert/helper.py b/pialert/helper.py index 5966bde8..9608dee7 100755 --- a/pialert/helper.py +++ b/pialert/helper.py @@ -37,6 +37,12 @@ def timeNowTZ(): def timeNow(): return datetime.datetime.now().replace(microsecond=0) +def get_timezone_offset(): + now = datetime.datetime.now(conf.tz) + offset_hours = now.utcoffset().total_seconds() / 3600 + offset_formatted = "{:+03d}:{:02d}".format(int(offset_hours), int((offset_hours % 1) * 60)) + return offset_formatted + #------------------------------------------------------------------------------- # App state @@ -470,58 +476,6 @@ def resolve_device_name_pholus (pMAC, pIP, allRes, nameNotFound, match_IP = Fals if 'PTR Class:IN' in value and len(value.split('"')) > 1: return cleanDeviceName(value.split('"')[1], match_IP) - - # # airplay matches contain a lot of information - # # Matches for example: - # # Brand Tv (50)._airplay._tcp.local. TXT Class:32769 "acl=0 deviceid=66:66:66:66:66:66 features=0x77777,0x38BCB46 rsf=0x3 fv=p20.T-FFFFFF-03.1 flags=0x204 model=XXXX manufacturer=Brand serialNumber=XXXXXXXXXXX protovers=1.1 srcvers=777.77.77 pi=FF:FF:FF:FF:FF:FF psi=00000000-0000-0000-0000-FFFFFFFFFF gid=00000000-0000-0000-0000-FFFFFFFFFF gcgl=0 pk=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" - # for i in pholusMatchesIndexes: - # if checkIPV4(allRes[i]['IP_v4_or_v6']) and '._airplay._tcp.local. TXT Class:32769' in str(allRes[i]["Value"]) : - # return cleanDeviceName(allRes[i]["Value"].split('._airplay._tcp.local. TXT Class:32769')[0], match_IP) - - # # second best - contains airplay - # # Matches for example: - # # _airplay._tcp.local. PTR Class:IN "Brand Tv (50)._airplay._tcp.local." - # for i in pholusMatchesIndexes: - # if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_airplay._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('._googlecast') not in allRes[i]["Value"]: - # return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP) - - # # Contains PTR Class:32769 - # # Matches for example: - # # 3.1.168.192.in-addr.arpa. PTR Class:32769 "MyPc.local." - # for i in pholusMatchesIndexes: - # if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:32769' in allRes[i]["Value"]: - # return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP) - - # # Contains AAAA Class:IN - # # Matches for example: - # # DESKTOP-SOMEID.local. AAAA Class:IN "fe80::fe80:fe80:fe80:fe80" - # for i in pholusMatchesIndexes: - # if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'AAAA Class:IN' in allRes[i]["Value"]: - # return cleanDeviceName(allRes[i]["Value"].split('.local.')[0], match_IP) - - # # Contains _googlecast._tcp.local. PTR Class:IN - # # Matches for example: - # # _googlecast._tcp.local. PTR Class:IN "Nest-Audio-ff77ff77ff77ff77ff77ff77ff77ff77._googlecast._tcp.local." - # for i in pholusMatchesIndexes: - # if checkIPV4(allRes[i]['IP_v4_or_v6']) and '_googlecast._tcp.local. PTR Class:IN' in allRes[i]["Value"] and ('Google-Cast-Group') not in allRes[i]["Value"]: - # return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP) - - # # Contains A Class:32769 - # # Matches for example: - # # Android.local. A Class:32769 "192.168.1.6" - # for i in pholusMatchesIndexes: - # if checkIPV4(allRes[i]['IP_v4_or_v6']) and ' A Class:32769' in allRes[i]["Value"]: - # return cleanDeviceName(allRes[i]["Value"].split(' A Class:32769')[0], match_IP) - - - # # # Contains PTR Class:IN - # # Matches for example: - # # _esphomelib._tcp.local. PTR Class:IN "ceiling-light-1._esphomelib._tcp.local." - # for i in pholusMatchesIndexes: - # if checkIPV4(allRes[i]['IP_v4_or_v6']) and 'PTR Class:IN' in allRes[i]["Value"]: - # if allRes[i]["Value"] and len(allRes[i]["Value"].split('"')) > 1: - # return cleanDeviceName(allRes[i]["Value"].split('"')[1], match_IP) - return nameNotFound diff --git a/pialert/networkscan.py b/pialert/networkscan.py index b80d15b6..1d336d3b 100755 --- a/pialert/networkscan.py +++ b/pialert/networkscan.py @@ -187,7 +187,7 @@ def insert_events (db): eve_PendingAlertEmail) SELECT dev_MAC, dev_LastIP, '{startTime}', 'Device Down', '', 1 FROM Devices - WHERE dev_AlertDeviceDown = 1 + WHERE dev_AlertDeviceDown != 0 AND dev_PresentLastScan = 1 AND NOT EXISTS (SELECT 1 FROM CurrentScan WHERE dev_MAC = cur_MAC diff --git a/pialert/notification.py b/pialert/notification.py index 7bcc9d4f..e3c52577 100755 --- a/pialert/notification.py +++ b/pialert/notification.py @@ -10,7 +10,7 @@ import conf import const from const import pialertPath, logPath, apiPath from logger import logResult, mylog, print_log -from helper import generate_mac_links, removeDuplicateNewLines, timeNowTZ, get_file_content, write_file +from helper import generate_mac_links, removeDuplicateNewLines, timeNowTZ, get_file_content, write_file, get_setting_value, get_timezone_offset #------------------------------------------------------------------------------- # Notification object handling @@ -209,7 +209,34 @@ class Notification_obj: self.save() - + def clearPendingEmailFlag(self): + + # Clean Pending Alert Events + self.db.sql.execute ("""UPDATE Devices SET dev_LastNotification = ? + WHERE dev_MAC IN ( + SELECT eve_MAC FROM Events + WHERE eve_PendingAlertEmail = 1 + ) + """, (timeNowTZ(),) ) + + self.db.sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0 + WHERE eve_PendingAlertEmail = 1 + AND eve_EventType !='Device Down' """) + + # Clear down events flag after the reporting window passed + self.db.sql.execute (f"""UPDATE Events SET eve_PendingAlertEmail = 0 + WHERE eve_PendingAlertEmail = 1 + AND eve_EventType =='Device Down' + AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}') + """) + + # clear plugin events + self.db.sql.execute ("DELETE FROM Plugins_Events") + + # DEBUG - print number of rows updated + mylog('minimal', ['[Notification] Notifications changes: ', self.db.sql.rowcount]) + + self.save() def save(self): # Commit changes diff --git a/pialert/reporting.py b/pialert/reporting.py index 88402839..db6de040 100755 --- a/pialert/reporting.py +++ b/pialert/reporting.py @@ -17,7 +17,7 @@ import json import conf import const from const import pialertPath, logPath, apiPath -from helper import timeNowTZ, get_file_content, write_file +from helper import timeNowTZ, get_file_content, write_file, get_timezone_offset, get_setting_value from logger import logResult, mylog, print_log @@ -74,10 +74,21 @@ def get_notifications (db): if 'down_devices' in conf.INCLUDED_SECTIONS : # Compose Devices Down Section - sqlQuery = """SELECT eve_MAC as MAC, eve_DateTime as Datetime, dev_LastIP as IP, eve_EventType as "Event Type", dev_Name as "Device name", dev_Comments as Comments FROM Events_Devices - WHERE eve_PendingAlertEmail = 1 - AND eve_EventType = 'Device Down' - ORDER BY eve_DateTime""" + sqlQuery = f""" + SELECT * + FROM Events AS down_events + WHERE eve_PendingAlertEmail = 1 + AND down_events.eve_EventType = 'Device Down' + AND eve_DateTime < datetime('now', '-{get_setting_value('NTFPRCS_alert_down_time')} minutes', '{get_timezone_offset()}') + AND NOT EXISTS ( + SELECT 1 + FROM Events AS connected_events + WHERE connected_events.eve_MAC = down_events.eve_MAC + AND connected_events.eve_EventType = 'Connected' + AND connected_events.eve_DateTime > down_events.eve_DateTime + ) + ORDER BY down_events.eve_DateTime; + """ # Get the events as JSON json_obj = db.get_table_as_json(sqlQuery) @@ -139,7 +150,8 @@ def skip_repeated_notifications (db): # Skip repeated notifications # due strfime : Overflow --> use "strftime / 60" - mylog('verbose','[Skip Repeated Notifications] Skip Repeated start') + mylog('verbose','[Skip Repeated Notifications] Skip Repeated') + db.sql.execute ("""UPDATE Events SET eve_PendingAlertEmail = 0 WHERE eve_PendingAlertEmail = 1 AND eve_MAC IN ( @@ -151,7 +163,7 @@ def skip_repeated_notifications (db): (strftime('%s','now','localtime')/60 ) ) """ ) - mylog('verbose','[Skip Repeated Notifications] Skip Repeated end') + db.commitDB()