vh/NetAlertX
1
0
Fork 0
mirror of https://github.com/jokob-sk/NetAlertX.git synced 2025-12-07 09:36:05 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Report fix + missing-in-last-scan functionality

Browse Source
This commit is contained in:
Jokob-sk
2023-08-20 11:26:58 +10:00
parent 75c5ec1215
commit 609e118a7a
11 changed files with 120 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
front/plugins/README.md
View File

@@ -489,6 +489,7 @@ You can have any `"function": "my_custom_name"` custom name, however, the ones l
- `new` means a new unique (unique combination of PrimaryId and SecondaryId) object was discovered.
- `watched-changed` - means that selected `Watched_ValueN` columns changed
- `watched-not-changed` - reports even on events where selected `Watched_ValueN` did not change
- `missing-in-last-scan` - if object is missing compared to previous scans
> 🔎 Example:

2
front/plugins/arp_scan/config.json
View File

@@ -184,7 +184,7 @@
"function": "REPORT_ON",
"type": "text.multiselect",
"default_value": ["new"],
"options": ["new", "watched-changed", "watched-not-changed"],
"options": ["new", "watched-changed", "watched-not-changed", "missing-in-last-scan"],
"localized": ["name", "description"],
"name": [
{

6
front/plugins/dhcp_leases/config.json
View File

@@ -213,6 +213,10 @@
{
"equals": "new",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
},
{
"equals": "missing-in-last-scan",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
}
],
"localized": ["name"],
@@ -321,7 +325,7 @@
"function": "REPORT_ON",
"type": "text.multiselect",
"default_value":["new","watched-changed"],
"options": ["new","watched-changed","watched-not-changed"],
"options": ["new","watched-changed","watched-not-changed", "missing-in-last-scan"],
"localized": ["name", "description"],
"name" :[{
"language_code":"en_us",

4
front/plugins/dhcp_servers/config.json
View File

@@ -181,6 +181,10 @@
{
"equals": "new",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
},
{
"equals": "missing-in-last-scan",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
}
],
"localized": ["name"],

4
front/plugins/nmap_services/config.json
View File

@@ -210,6 +210,10 @@
{
"equals": "new",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
},
{
"equals": "missing-in-last-scan",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
}
],
"localized": ["name"],

2
front/plugins/pihole_scan/config.json
View File

@@ -121,7 +121,7 @@
"function": "REPORT_ON",
"type": "text.multiselect",
"default_value": ["new"],
"options": ["new", "watched-changed", "watched-not-changed"],
"options": ["new", "watched-changed", "watched-not-changed", "missing-in-last-scan"],
"localized": ["name", "description"],
"name": [
{

6
front/plugins/snmp_discovery/config.json
View File

@@ -214,6 +214,10 @@
{
"equals": "new",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
},
{
"equals": "missing-in-last-scan",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
}
],
"localized": ["name"],
@@ -322,7 +326,7 @@
"function": "REPORT_ON",
"type": "text.multiselect",
"default_value":["new","watched-changed"],
"options": ["new","watched-changed","watched-not-changed"],
"options": ["new","watched-changed","watched-not-changed", "missing-in-last-scan"],
"localized": ["name", "description"],
"name" :[{
"language_code":"en_us",

2
front/plugins/undiscoverables/config.json
View File

@@ -153,7 +153,7 @@
"function": "REPORT_ON",
"type": "readonly",
"default_value": [],
"options": ["new", "watched-changed", "watched-not-changed"],
"options": ["new", "watched-changed", "watched-not-changed", "missing-in-last-scan"],
"localized": ["name", "description"],
"name": [
{

6
front/plugins/unifi_import/config.json
View File

@@ -244,6 +244,10 @@
{
"equals": "new",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
},
{
"equals": "missing-in-last-scan",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
}
],
"localized": ["name"],
@@ -442,7 +446,7 @@
"function": "REPORT_ON",
"type": "text.multiselect",
"default_value":["new","watched-changed"],
"options": ["new","watched-changed","watched-not-changed"],
"options": ["new","watched-changed","watched-not-changed", "missing-in-last-scan"],
"localized": ["name", "description"],
"name" :[{
"language_code":"en_us",

6
front/plugins/website_monitor/config.json
View File

@@ -272,6 +272,10 @@
{
"equals": "new",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-circle-plus'></i></div>"
},
{
"equals": "missing-in-last-scan",
"replacement": "<div style='text-align:center'><i class='fa-solid fa-question'></i></div>"
}
],
"localized": ["name"],
@@ -449,7 +453,7 @@
"function": "REPORT_ON",
"type": "text.multiselect",
"default_value":["new","watched-changed"],
"options": ["new","watched-changed","watched-not-changed"],
"options": ["new","watched-changed","watched-not-changed", "missing-in-last-scan"],
"localized": ["name", "description"],
"name" :[{
"language_code": "en_us",

112
pialert/plugin.py
View File

@@ -232,9 +232,12 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
columns = line.split("|")
# There has to be always 9 columns
if len(columns) == 9:
# Construct a tuple of values to be inserted into the database table.
# Create a tuple containing values to be inserted into the database.
# Each value corresponds to a column in the table in the order of the columns.
# must match the Plugins_Objects and Plugins_Events databse tables and can be used as input for the plugin_object_class.
sqlParams.append(
(
0, # "Index" placeholder
plugin["unique_prefix"], # "Plugin" column value from the plugin dictionary
columns[0], # "Object_PrimaryID" value from columns list
columns[1], # "Object_SecondaryID" value from columns list
@@ -244,7 +247,7 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
columns[4], # "Watched_Value2" value from columns list
columns[5], # "Watched_Value3" value from columns list
columns[6], # "Watched_Value4" value from columns list
0, # Placeholder for "Status" column
'not-processed', # "Status" column (placeholder)
columns[7], # "Extra" value from columns list
'null', # Placeholder for "UserData" column
columns[8] # "ForeignKey" value from columns list
@@ -269,9 +272,12 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
for row in arr:
# There has to be always 9 columns
if len(row) == 9 and (row[0] in ['','null']) == False :
# Construct a tuple of values to be inserted into the database table.
# Create a tuple containing values to be inserted into the database.
# Each value corresponds to a column in the table in the order of the columns.
# must match the Plugins_Objects and Plugins_Events databse tables and can be used as input for the plugin_object_class
sqlParams.append(
(
0, # "Index" placeholder
plugin["unique_prefix"], # "Plugin" plugin dictionary
row[0], # "Object_PrimaryID" row
handle_empty(row[1]), # "Object_SecondaryID" column after handling empty values
@@ -281,7 +287,7 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
row[4], # "Watched_Value2" row
handle_empty(row[5]), # "Watched_Value3" column after handling empty values
handle_empty(row[6]), # "Watched_Value4" column after handling empty values
0, # Placeholder "Status" column
'not-processed', # "Status" column (placeholder)
row[7], # "Extra" row
'null', # Placeholder "UserData" column
row[8] # "ForeignKey" row
@@ -322,7 +328,9 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
if len(row) == 9 and (row[0] in ['','null']) == False :
# Create a tuple containing values to be inserted into the database.
# Each value corresponds to a column in the table in the order of the columns.
# must match the Plugins_Objects and Plugins_Events databse tables and can be used as input for the plugin_object_class
sqlParams.append(
0, # "Index" placeholder
(plugin["unique_prefix"], # "Plugin"
row[0], # "Object_PrimaryID"
handle_empty(row[1]), # "Object_SecondaryID"
@@ -332,7 +340,7 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
row[4], # "Watched_Value2"
handle_empty(row[5]), # "Watched_Value3"
handle_empty(row[6]), # "Watched_Value4"
0, # "Status" column (placeholder)
'not-processed', # "Status" column (placeholder)
row[7], # "Extra"
'null', # "UserData" column (null placeholder)
row[8])) # "ForeignKey"
@@ -349,11 +357,9 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
# process results if any
if len(sqlParams) > 0:
sql.executemany ("""INSERT INTO Plugins_Events ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams)
db.commitDB()
try:
sql.executemany("""INSERT INTO Plugins_History ("Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams)
sql.executemany("""INSERT INTO Plugins_History ("Index", "Plugin", "Object_PrimaryID", "Object_SecondaryID", "DateTimeCreated", "DateTimeChanged", "Watched_Value1", "Watched_Value2", "Watched_Value3", "Watched_Value4", "Status" ,"Extra", "UserData", "ForeignKey") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", sqlParams)
db.commitDB()
except sqlite3.Error as e:
db.rollbackDB() # Rollback changes in case of an error
@@ -361,7 +367,7 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
# create objects
pluginsState = process_plugin_events(db, plugin, pluginsState)
pluginsState = process_plugin_events(db, plugin, pluginsState, sqlParams)
# update API endpoints
update_api(db, False, ["plugins_events","plugins_objects"])
@@ -497,7 +503,7 @@ def combine_plugin_objects(old, new):
#-------------------------------------------------------------------------------
# Check if watched values changed for the given plugin
def process_plugin_events(db, plugin, pluginsState):
def process_plugin_events(db, plugin, pluginsState, plugEventsArr):
sql = db.sql
# Access the connection from the DB instance
@@ -511,8 +517,8 @@ def process_plugin_events(db, plugin, pluginsState):
# Begin a transaction
with conn:
# get existing objects
plugObjectsArr = db.get_sql_array ("SELECT * FROM Plugins_Objects where Plugin = '" + str(pluginPref)+"'")
plugEventsArr = db.get_sql_array ("SELECT * FROM Plugins_Events where Plugin = '" + str(pluginPref)+"'")
pluginObjects = []
pluginEvents = []
@@ -525,46 +531,65 @@ def process_plugin_events(db, plugin, pluginsState):
mylog('debug', ['[Plugins] Existing objects : ', existingPluginObjectsCount])
mylog('debug', ['[Plugins] New and existing events : ', len(plugEventsArr)])
# set status as new - will be changed later if conditions are fulfilled, e.g. entry found
# create plugin objects from events - will be processed to find existing objects
for eve in plugEventsArr:
tmpObject = plugin_object_class(plugin, eve)
tmpObject.status = "new"
pluginEvents.append(tmpObject)
pluginEvents.append(plugin_object_class(plugin, eve))
# Update the status to "exists"
# Loop thru all current events and update the status to "exists" if the event matches an existing object
index = 0
for tmpObjFromEvent in pluginEvents:
# compare hash of the IDs for uniqueness
if any(x.idsHash == tmpObject.idsHash for x in pluginObjects):
# mylog('debug', ['[Plugins] Found existing object'])
if any(x.idsHash == tmpObjFromEvent.idsHash for x in pluginObjects):
pluginEvents[index].status = "exists"
index += 1
# Loop thru events and update the one that exist to determine if watched columns changed
# Loop thru events and check if the ones that exist have changed in the watched columns
# if yes update status accordingly
index = 0
for tmpObjFromEvent in pluginEvents:
if tmpObjFromEvent.status == "exists":
# compare hash of the changed watched columns for uniqueness
if any(x.watchedHash != tmpObject.watchedHash for x in pluginObjects):
if any(x.watchedHash != tmpObjFromEvent.watchedHash for x in pluginObjects):
pluginEvents[index].status = "watched-changed"
else:
pluginEvents[index].status = "watched-not-changed"
index += 1
# Loop thru events and check if previously available objects are missing
# Create a set of hashes of IDs in pluginObjects
plugin_objects_ids_hash = set(x.idsHash for x in pluginObjects)
for tmpObjFromEvent in pluginEvents:
if tmpObjFromEvent.idsHash not in plugin_objects_ids_hash:
for x in pluginObjects:
if x.primaryId == tmpObjFromEvent.primaryId and x.secondaryId == tmpObjFromEvent.secondaryId:
x.status = "missing-in-last-scan"
x.changed = datetime.now(timeZone).strftime("%Y-%m-%d %H:%M:%S")
break
# Merge existing plugin objects with newly discovered ones and update existing ones with new values
for eveObj in pluginEvents:
if eveObj.status == 'new':
pluginObjects.append(eveObj)
for tmpObjFromEvent in pluginEvents:
# set "new" status for new objects and append
if tmpObjFromEvent.status == 'not-processed':
# This is a new object as it was not discovered as "exists" previously
tmpObjFromEvent.status = 'new'
pluginObjects.append(tmpObjFromEvent)
# update data of existing objects
else:
index = 0
for plugObj in pluginObjects:
# find corresponding object for the event and merge
if plugObj.idsHash == eveObj.idsHash:
pluginObjects[index] = combine_plugin_objects(plugObj, eveObj)
if plugObj.idsHash == tmpObjFromEvent.idsHash:
pluginObjects[index] = combine_plugin_objects(plugObj, tmpObjFromEvent)
index += 1
@@ -575,8 +600,14 @@ def process_plugin_events(db, plugin, pluginsState):
objects_to_insert = []
events_to_insert = []
statuses_to_report_on = get_plugin_setting_value(plugin, "REPORT_ON")
mylog('debug', ['[Plugins] statuses_to_report_on: ', statuses_to_report_on])
for plugObj in pluginObjects:
# keep old createdTime time if the plugObj already was created before
createdTime = plugObj.changed if plugObj.status == 'new' else plugObj.created
values = (
plugObj.pluginPref, plugObj.primaryId, plugObj.secondaryId, createdTime,
plugObj.changed, plugObj.watched1, plugObj.watched2, plugObj.watched3,
@@ -589,11 +620,16 @@ def process_plugin_events(db, plugin, pluginsState):
else:
objects_to_insert.append(values + (plugObj.index,)) # Include index for UPDATE
if plugObj.status == 'new':
events_to_insert.append(values)
elif plugObj.status in get_plugin_setting_value(plugin, "REPORT_ON"):
# only generate events that we want to be notified on
if plugObj.status in statuses_to_report_on:
events_to_insert.append(values)
mylog('debug', ['[Plugins] events_to_insert count: ', len(events_to_insert)])
mylog('debug', ['[Plugins] pluginEvents count : ', len(pluginEvents)])
logEventStatusCounts(pluginEvents)
mylog('debug', ['[Plugins] pluginObjects count: ', len(pluginObjects)])
logEventStatusCounts(pluginObjects)
# Bulk insert/update objects
if objects_to_insert:
sql.executemany(
@@ -732,8 +768,8 @@ class plugin_object_class:
self.pluginPref = objDbRow[1]
self.primaryId = objDbRow[2]
self.secondaryId = objDbRow[3]
self.created = objDbRow[4]
self.changed = objDbRow[5]
self.created = objDbRow[4] # can be null
self.changed = objDbRow[5] # never null (data coming from plugin)
self.watched1 = objDbRow[6]
self.watched2 = objDbRow[7]
self.watched3 = objDbRow[8]
@@ -743,6 +779,10 @@ class plugin_object_class:
self.userData = objDbRow[12]
self.foreignKey = objDbRow[13]
# Check if self.status is valid
if self.status not in ["exists", "watched-changed", "watched-not-changed", "new", "not-processed", "missing-in-last-scan"]:
raise ValueError("Invalid status value for plugin object:", self.status)
# self.idsHash = str(hash(str(self.primaryId) + str(self.secondaryId)))
self.idsHash = str(self.primaryId) + str(self.secondaryId)
@@ -768,4 +808,16 @@ class plugin_object_class:
self.watchedHash = str(hash(tmp))
#-------------------------------------------------------------------------------
def logEventStatusCounts(pluginEvents):
status_counts = {} # Dictionary to store counts for each status
for event in pluginEvents:
status = event.status
if status in status_counts:
status_counts[status] += 1
else:
status_counts[status] = 1
for status, count in status_counts.items():
mylog('debug', [f'Status "{status}": {count} events'])