From ff9245c31dc5a139b3582923c82685d73a36fdcd Mon Sep 17 00:00:00 2001 From: Jokob-sk Date: Mon, 7 Aug 2023 15:33:41 +1000 Subject: [PATCH] ARPSCAN to plugin rewrite --- front/plugins/README.md | 2 +- front/plugins/arp_scan/config.json | 12 +- front/plugins/arp_scan/script.py | 41 +++++- front/plugins/snmp_discovery/config.json | 2 +- pialert/__main__.py | 12 +- pialert/database.py | 2 +- pialert/networkscan.py | 4 +- pialert/plugin.py | 165 ++++++++++++++--------- 8 files changed, 161 insertions(+), 79 deletions(-) diff --git a/front/plugins/README.md b/front/plugins/README.md index 9a33e305..a45a7a99 100755 --- a/front/plugins/README.md +++ b/front/plugins/README.md @@ -63,7 +63,7 @@ UI displays outdated values until the API endpoints get refreshed. ## Plugin file structure overview -> Folder name must be the same as the code name value in: `"code_name": ""` +> ⚠️Folder name must be the same as the code name value in: `"code_name": ""` > Unique prefix needs to be unique compared to the other settings prefixes, e.g.: the prefix `APPRISE` is already in use. | File | Required (plugin type) | Description | diff --git a/front/plugins/arp_scan/config.json b/front/plugins/arp_scan/config.json index 0c9b53ab..e45dacca 100755 --- a/front/plugins/arp_scan/config.json +++ b/front/plugins/arp_scan/config.json @@ -1,10 +1,18 @@ { - "code_name": "arpscan", + "code_name": "arp_scan", "unique_prefix": "ARPSCAN", "enabled": true, "data_source": "script", "mapped_to_table": "CurrentScan", - + "data_filters": [ + { + "compare_column" : "Object_PrimaryID", + "compare_operator" : "==", + "compare_field_id": "txtMacFilter", + "compare_js_template": "'{value}'.toString()", + "compare_use_quotes": true + } + ], "localized": ["display_name", "description", "icon"], "display_name": [ diff --git a/front/plugins/arp_scan/script.py b/front/plugins/arp_scan/script.py index 5e16ee96..7ebb4a92 100755 --- a/front/plugins/arp_scan/script.py +++ b/front/plugins/arp_scan/script.py @@ -5,6 +5,7 @@ import pathlib import argparse import sys import re +import base64 import subprocess from time import strftime @@ -18,22 +19,54 @@ RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log') def main(): + # sample + # /home/pi/pialert/front/plugins/arp_scan/script.py userSubnets=b'MTkyLjE2OC4xLjAvMjQgLS1pbnRlcmZhY2U9ZXRoMQ==' # the script expects a parameter in the format of userSubnets=subnet1,subnet2,... parser = argparse.ArgumentParser(description='Import devices from settings') parser.add_argument('userSubnets', nargs='+', help="list of subnets with options") values = parser.parse_args() + import base64 + + # Assuming Plugin_Objects is a class or function that reads data from the RESULT_FILE + # and returns a list of objects called 'devices'. devices = Plugin_Objects(RESULT_FILE) - subnets_list = [] + # Print a message to indicate that the script is starting. + print('In script:') - if isinstance(values.userSubnets, list): - subnets_list = values.userSubnets + # Assuming 'values' is a dictionary or object that contains a key 'userSubnets' + # which holds a list of user-submitted subnets. + # Printing the userSubnets list to check its content. + print(values.userSubnets) + + # Extract the base64-encoded subnet information from the first element of the userSubnets list. + # The format of the element is assumed to be like 'userSubnets=b'. + userSubnetsParamBase64 = values.userSubnets[0].split('userSubnets=b')[1] + + # Printing the extracted base64-encoded subnet information. + print(userSubnetsParamBase64) + + # Decode the base64-encoded subnet information to get the actual subnet information in ASCII format. + userSubnetsParam = base64.b64decode(userSubnetsParamBase64).decode('ascii') + + # Print the decoded subnet information. + print('userSubnetsParam:') + print(userSubnetsParam) + + # Check if the decoded subnet information contains multiple subnets separated by commas. + # If it does, split the string into a list of individual subnets. + # Otherwise, create a list with a single element containing the subnet information. + if ',' in userSubnetsParam: + subnets_list = userSubnetsParam.split(',') else: - subnets_list = [values.userSubnets] + subnets_list = [userSubnetsParam] + # Execute the ARP scanning process on the list of subnets (whether it's one or multiple subnets). + # The function 'execute_arpscan' is assumed to be defined elsewhere in the code. unique_devices = execute_arpscan(subnets_list) + for device in unique_devices: devices.add_object( primaryId=device['mac'], # MAC (Device Name) diff --git a/front/plugins/snmp_discovery/config.json b/front/plugins/snmp_discovery/config.json index 6b99c0bc..9a3b4f09 100755 --- a/front/plugins/snmp_discovery/config.json +++ b/front/plugins/snmp_discovery/config.json @@ -2,7 +2,7 @@ "code_name": "snmp_discovery", "unique_prefix": "SNMPDSC", "enabled": true, - "data_source": "pyton-script", + "data_source": "script", "data_filters": [ { "compare_column" : "Object_PrimaryID", diff --git a/pialert/__main__.py b/pialert/__main__.py index 5be82db0..c24bf19d 100755 --- a/pialert/__main__.py +++ b/pialert/__main__.py @@ -107,7 +107,14 @@ def main (): # update time started conf.loop_start_time = timeNowTZ() + + # TODO fix these loop_start_time = conf.loop_start_time # TODO fix + last_update_vendors = conf.last_update_vendors + last_network_scan = conf.last_network_scan + last_cleanup = conf.last_cleanup + last_version_check = conf.last_version_check + # check if new version is available / only check once an hour if conf.last_version_check + datetime.timedelta(hours=1) < loop_start_time : @@ -128,10 +135,11 @@ def main (): update_api(db) # proceed if 1 minute passed - if last_scan_run + datetime.timedelta(minutes=1) < loop_start_time : + if conf.last_scan_run + datetime.timedelta(minutes=1) < conf.loop_start_time : # last time any scan or maintenance/upkeep was run - last_scan_run = loop_start_time + conf.last_scan_run = loop_start_time + last_internet_IP_scan = conf.last_internet_IP_scan # Header updateState(db,"Process: Start") diff --git a/pialert/database.py b/pialert/database.py index 5c4073c5..437b9ea2 100755 --- a/pialert/database.py +++ b/pialert/database.py @@ -397,7 +397,7 @@ class DB(): self.sql.execute("DROP TABLE CurrentScan;") self.sql.execute(""" CREATE TABLE CurrentScan ( - cur_ScanCycle INTEGER NOT NULL, + cur_ScanCycle INTEGER, cur_MAC STRING(50) NOT NULL COLLATE NOCASE, cur_IP STRING(50) NOT NULL COLLATE NOCASE, cur_Vendor STRING(250), diff --git a/pialert/networkscan.py b/pialert/networkscan.py index c731a16e..d18c0179 100755 --- a/pialert/networkscan.py +++ b/pialert/networkscan.py @@ -91,8 +91,8 @@ def process_scan (db): mylog('verbose','[Process Scan] Skipping repeated notifications') skip_repeated_notifications (db) - # Clear current scan as processed TODO uncomment - # db.sql.execute ("DELETE FROM CurrentScan") + # Clear current scan as processed + db.sql.execute ("DELETE FROM CurrentScan") # Commit changes db.commitDB() diff --git a/pialert/plugin.py b/pialert/plugin.py index 1caaac9b..e2b9c69d 100755 --- a/pialert/plugin.py +++ b/pialert/plugin.py @@ -2,6 +2,7 @@ import os import json import subprocess import datetime +import base64 from collections import namedtuple # pialert modules @@ -229,7 +230,7 @@ def execute_plugin(db, plugin): if len(columns) == 9: sqlParams.append((plugin["unique_prefix"], columns[0], columns[1], 'null', columns[2], columns[3], columns[4], columns[5], columns[6], 0, columns[7], 'null', columns[8])) else: - mylog('none', ['[Plugins]: Skipped invalid line in the output: ', line]) + mylog('none', ['[Plugins] Skipped invalid line in the output: ', line]) else: mylog('debug', [f'[Plugins] The file {file_path} does not exist']) @@ -249,7 +250,7 @@ def execute_plugin(db, plugin): if len(row) == 9 and (row[0] in ['','null']) == False : sqlParams.append((plugin["unique_prefix"], row[0], handle_empty(row[1]), 'null', row[2], row[3], row[4], handle_empty(row[5]), handle_empty(row[6]), 0, row[7], 'null', row[8])) else: - mylog('none', ['[Plugins]: Skipped invalid sql result']) + mylog('none', ['[Plugins] Skipped invalid sql result']) # check if the subprocess / SQL query failed / there was no valid output @@ -257,7 +258,7 @@ def execute_plugin(db, plugin): mylog('none', ['[Plugins] No output received from the plugin ', plugin["unique_prefix"], ' - enable LOG_LEVEL=debug and check logs']) return else: - mylog('verbose', ['[Plugins]: SUCCESS, received ', len(sqlParams), ' entries']) + mylog('verbose', ['[Plugins] SUCCESS, received ', len(sqlParams), ' entries']) # process results if any if len(sqlParams) > 0: @@ -293,20 +294,27 @@ def passable_string_from_setting(globalSetting): noConversion = ['text', 'string', 'integer', 'boolean', 'password', 'readonly', 'integer.select', 'text.select', 'integer.checkbox' ] - arrayConversion = ['text.multiselect', 'list', 'subnets'] + arrayConversion = ['text.multiselect', 'list'] + arrayConversionBase64 = ['subnets'] jsonConversion = ['.template'] + mylog('debug', f'[Plugins] setTyp: {setTyp}') + if setTyp in noConversion: return setVal if setTyp in arrayConversion: return flatten_array(setVal) + if setTyp in arrayConversionBase64: + + return flatten_array(setVal, encodeBase64 = True) + for item in jsonConversion: if setTyp.endswith(item): return json.dumps(setVal) - mylog('none', ['[Plugins]: ERROR: Parameter not converted.']) + mylog('none', ['[Plugins] ERROR: Parameter not converted.']) @@ -337,33 +345,47 @@ def get_setting_value(key): return '' #------------------------------------------------------------------------------- -def flatten_array(arr): - +def flatten_array(arr, encodeBase64=False): tmp = '' + arrayItemStr = '' + mylog('debug', '[Plugins] Flattening the below array') + mylog('debug', f'[Plugins] Convert to Base64: {encodeBase64}') mylog('debug', arr) - for arrayItem in arr: + for arrayItem in arr: # only one column flattening is supported if isinstance(arrayItem, list): - arrayItem = str(arrayItem[0]) + arrayItemStr = str(arrayItem[0]).replace("'", '') # removing single quotes - not allowed + else: + # is string already + arrayItemStr = arrayItem - tmp += arrayItem + ',' - # tmp = tmp.replace("'","").replace(' ','') # No single quotes or empty spaces allowed - tmp = tmp.replace("'","") # No single quotes allowed - return tmp[:-1] # Remove last comma ',' + tmp += f'{arrayItemStr},' + + tmp = tmp[:-1] # Remove last comma ',' + + mylog('debug', f'[Plugins] Flattened array: {tmp}') + + if encodeBase64: + tmp = str(base64.b64encode(tmp.encode('ascii'))) + mylog('debug', f'[Plugins] Flattened array (base64): {tmp}') + + + return tmp + #------------------------------------------------------------------------------- # Replace {wildcars} with parameters def resolve_wildcards_arr(commandArr, params): - mylog('debug', ['[Plugins]: Pre-Resolved CMD: '] + commandArr) + mylog('debug', ['[Plugins] Pre-Resolved CMD: '] + commandArr) for param in params: - # mylog('debug', ['[Plugins]: key : {', param[0], '}']) - # mylog('debug', ['[Plugins]: resolved: ', param[1]]) + # mylog('debug', ['[Plugins] key : {', param[0], '}']) + # mylog('debug', ['[Plugins] resolved: ', param[1]]) i = 0 @@ -493,67 +515,78 @@ def process_plugin_events(db, plugin): # Perform databse table mapping if enabled for the plugin if len(pluginEvents) > 0 and "mapped_to_table" in plugin: - sqlParams = [] + # Initialize an empty list to store SQL parameters. + sqlParams = [] - dbTable = plugin['mapped_to_table'] + # Get the database table name from the 'mapped_to_table' key in the 'plugin' dictionary. + dbTable = plugin['mapped_to_table'] - mylog('debug', ['[Plugins] Mapping objects to database table: ', dbTable]) + # Log a debug message indicating the mapping of objects to the database table. + mylog('debug', ['[Plugins] Mapping objects to database table: ', dbTable]) - # collect all columns to be mapped - mappedCols = [] - columnsStr = '' - valuesStr = '' + # Initialize lists to hold mapped column names, columnsStr, and valuesStr for SQL query. + mappedCols = [] + columnsStr = '' + valuesStr = '' - for clmn in plugin['database_column_definitions']: - if 'mapped_to_column' in clmn: - mappedCols.append(clmn) - columnsStr = f'{columnsStr}, "{clmn["mapped_to_column"]}"' - valuesStr = f'{valuesStr}, ?' + # Loop through the 'database_column_definitions' in the 'plugin' dictionary to collect mapped columns. + # Build the columnsStr and valuesStr for the SQL query. + for clmn in plugin['database_column_definitions']: + if 'mapped_to_column' in clmn: + mappedCols.append(clmn) + columnsStr = f'{columnsStr}, "{clmn["mapped_to_column"]}"' + valuesStr = f'{valuesStr}, ?' - if len(columnsStr) > 0: - columnsStr = columnsStr[1:] # remove first ',' - valuesStr = valuesStr[1:] # remove first ',' + # Remove the first ',' from columnsStr and valuesStr. + if len(columnsStr) > 0: + columnsStr = columnsStr[1:] + valuesStr = valuesStr[1:] - # map the column names to plugin object event values - for plgEv in pluginEvents: + # Map the column names to plugin object event values and create a list of tuples 'sqlParams'. + for plgEv in pluginEvents: + tmpList = [] - tmpList = [] + for col in mappedCols: + if col['column'] == 'Index': + tmpList.append(plgEv.index) + elif col['column'] == 'Plugin': + tmpList.append(plgEv.pluginPref) + elif col['column'] == 'Object_PrimaryID': + tmpList.append(plgEv.primaryId) + elif col['column'] == 'Object_SecondaryID': + tmpList.append(plgEv.secondaryId) + elif col['column'] == 'DateTimeCreated': + tmpList.append(plgEv.created) + elif col['column'] == 'DateTimeChanged': + tmpList.append(plgEv.changed) + elif col['column'] == 'Watched_Value1': + tmpList.append(plgEv.watched1) + elif col['column'] == 'Watched_Value2': + tmpList.append(plgEv.watched2) + elif col['column'] == 'Watched_Value3': + tmpList.append(plgEv.watched3) + elif col['column'] == 'Watched_Value4': + tmpList.append(plgEv.watched4) + elif col['column'] == 'UserData': + tmpList.append(plgEv.userData) + elif col['column'] == 'Extra': + tmpList.append(plgEv.extra) + elif col['column'] == 'Status': + tmpList.append(plgEv.status) - for col in mappedCols: - if col['column'] == 'Index': - tmpList.append(plgEv.index) - elif col['column'] == 'Plugin': - tmpList.append(plgEv.pluginPref) - elif col['column'] == 'Object_PrimaryID': - tmpList.append(plgEv.primaryId) - elif col['column'] == 'Object_SecondaryID': - tmpList.append(plgEv.secondaryId) - elif col['column'] == 'DateTimeCreated': - tmpList.append(plgEv.created) - elif col['column'] == 'DateTimeChanged': - tmpList.append(plgEv.changed) - elif col['column'] == 'Watched_Value1': - tmpList.append(plgEv.watched1) - elif col['column'] == 'Watched_Value2': - tmpList.append(plgEv.watched2) - elif col['column'] == 'Watched_Value3': - tmpList.append(plgEv.watched3) - elif col['column'] == 'Watched_Value4': - tmpList.append(plgEv.watched4) - elif col['column'] == 'UserData': - tmpList.append(plgEv.userData) - elif col['column'] == 'Extra': - tmpList.append(plgEv.extra) - elif col['column'] == 'Status': - tmpList.append(plgEv.status) - - sqlParams.append(tuple(tmpList)) + # Append the mapped values to the list 'sqlParams' as a tuple. + sqlParams.append(tuple(tmpList)) - q = f'INSERT into {dbTable} ({columnsStr}) VALUES ({valuesStr})' + # Generate the SQL INSERT query using the collected information. + q = f'INSERT into {dbTable} ({columnsStr}) VALUES ({valuesStr})' - mylog('debug', ['[Plugins] SQL query for mapping: ', q ]) + # Log a debug message showing the generated SQL query for mapping. + mylog('debug', ['[Plugins] SQL query for mapping: ', q]) + + # Execute the SQL query using 'sql.executemany()' and the 'sqlParams' list of tuples. + # This will insert multiple rows into the database in one go. + sql.executemany(q, sqlParams) - sql.executemany (q, sqlParams) db.commitDB()