ARPSCAN to plugin rewrite

front/plugins/README.md

@@ -63,7 +63,7 @@ UI displays outdated values until the API endpoints get refreshed.
 
 ## Plugin file structure overview 
 
-> Folder name must be the same as the code name value in: `"code_name": "<value>"`
+> ⚠️Folder name must be the same as the code name value in: `"code_name": "<value>"`
 > Unique prefix needs to be unique compared to the other settings prefixes, e.g.: the prefix `APPRISE` is already in use. 
 
   | File | Required (plugin type) | Description | 

front/plugins/arp_scan/config.json

@@ -1,10 +1,18 @@
 {
-  "code_name": "arpscan",
+  "code_name": "arp_scan",
   "unique_prefix": "ARPSCAN",
   "enabled": true,
   "data_source": "script",
   "mapped_to_table": "CurrentScan", 
-
+  "data_filters": [
+    {
+        "compare_column" : "Object_PrimaryID",
+        "compare_operator" : "==",
+        "compare_field_id": "txtMacFilter",
+        "compare_js_template": "'{value}'.toString()", 
+        "compare_use_quotes": true 
+    }
+  ],
   "localized": ["display_name", "description", "icon"],
 
   "display_name": [

front/plugins/arp_scan/script.py

@@ -5,6 +5,7 @@ import pathlib
 import argparse
 import sys
 import re
+import base64
 import subprocess
 from time import strftime
 
@@ -18,22 +19,54 @@ RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log')
 
 
 def main():
+    # sample
+    # /home/pi/pialert/front/plugins/arp_scan/script.py userSubnets=b'MTkyLjE2OC4xLjAvMjQgLS1pbnRlcmZhY2U9ZXRoMQ=='
     # the script expects a parameter in the format of userSubnets=subnet1,subnet2,...
     parser = argparse.ArgumentParser(description='Import devices from settings')
     parser.add_argument('userSubnets', nargs='+', help="list of subnets with options")
     values = parser.parse_args()
 
+    import base64
+
+    # Assuming Plugin_Objects is a class or function that reads data from the RESULT_FILE
+    # and returns a list of objects called 'devices'.
     devices = Plugin_Objects(RESULT_FILE)
 
-    subnets_list = []
+    # Print a message to indicate that the script is starting.
+    print('In script:')
 
-    if isinstance(values.userSubnets, list):
+    # Assuming 'values' is a dictionary or object that contains a key 'userSubnets'
-        subnets_list = values.userSubnets
+    # which holds a list of user-submitted subnets.
+    # Printing the userSubnets list to check its content.
+    print(values.userSubnets)
+
+    # Extract the base64-encoded subnet information from the first element of the userSubnets list.
+    # The format of the element is assumed to be like 'userSubnets=b<base64-encoded-data>'.
+    userSubnetsParamBase64 = values.userSubnets[0].split('userSubnets=b')[1]
+
+    # Printing the extracted base64-encoded subnet information.
+    print(userSubnetsParamBase64)
+
+    # Decode the base64-encoded subnet information to get the actual subnet information in ASCII format.
+    userSubnetsParam = base64.b64decode(userSubnetsParamBase64).decode('ascii')
+
+    # Print the decoded subnet information.
+    print('userSubnetsParam:')
+    print(userSubnetsParam)
+
+    # Check if the decoded subnet information contains multiple subnets separated by commas.
+    # If it does, split the string into a list of individual subnets.
+    # Otherwise, create a list with a single element containing the subnet information.
+    if ',' in userSubnetsParam:
+        subnets_list = userSubnetsParam.split(',')
     else:
-        subnets_list = [values.userSubnets]
+        subnets_list = [userSubnetsParam]
 
+    # Execute the ARP scanning process on the list of subnets (whether it's one or multiple subnets).
+    # The function 'execute_arpscan' is assumed to be defined elsewhere in the code.
     unique_devices = execute_arpscan(subnets_list)
 
+
     for device in unique_devices:
         devices.add_object(
             primaryId=device['mac'],  # MAC (Device Name)

front/plugins/snmp_discovery/config.json

@@ -2,7 +2,7 @@
     "code_name": "snmp_discovery",
     "unique_prefix": "SNMPDSC",
     "enabled": true,    
-    "data_source":  "pyton-script",
+    "data_source":  "script",
     "data_filters": [
         {
             "compare_column" : "Object_PrimaryID",

pialert/__main__.py

@@ -107,7 +107,14 @@ def main ():
 
         # update time started
         conf.loop_start_time = timeNowTZ()
+        
+        # TODO fix these
         loop_start_time = conf.loop_start_time # TODO fix
+        last_update_vendors = conf.last_update_vendors
+        last_network_scan = conf.last_network_scan
+        last_cleanup = conf.last_cleanup
+        last_version_check = conf.last_version_check
+        
 
         # check if new version is available / only check once an hour
         if conf.last_version_check  + datetime.timedelta(hours=1) < loop_start_time :
@@ -128,10 +135,11 @@ def main ():
         update_api(db)
         
         # proceed if 1 minute passed
-        if last_scan_run + datetime.timedelta(minutes=1) < loop_start_time :
+        if conf.last_scan_run + datetime.timedelta(minutes=1) < conf.loop_start_time :
 
              # last time any scan or maintenance/upkeep was run
-            last_scan_run = loop_start_time
+            conf.last_scan_run = loop_start_time
+            last_internet_IP_scan = conf.last_internet_IP_scan
 
             # Header
             updateState(db,"Process: Start")      

pialert/database.py

@@ -397,7 +397,7 @@ class DB():
             self.sql.execute("DROP TABLE CurrentScan;")
 
         self.sql.execute(""" CREATE TABLE CurrentScan (
-                                cur_ScanCycle INTEGER NOT NULL,
+                                cur_ScanCycle INTEGER,
                                 cur_MAC STRING(50) NOT NULL COLLATE NOCASE,
                                 cur_IP STRING(50) NOT NULL COLLATE NOCASE,
                                 cur_Vendor STRING(250),

pialert/networkscan.py

@@ -91,8 +91,8 @@ def process_scan (db):
     mylog('verbose','[Process Scan] Skipping repeated notifications')
     skip_repeated_notifications (db)
 
-    # Clear current scan as processed TODO uncomment
+    # Clear current scan as processed 
-    # db.sql.execute ("DELETE FROM CurrentScan")
+    db.sql.execute ("DELETE FROM CurrentScan")
   
     # Commit changes    
     db.commitDB()

pialert/plugin.py

@@ -2,6 +2,7 @@ import os
 import json
 import subprocess
 import datetime
+import base64
 from collections import namedtuple
 
 # pialert modules
@@ -229,7 +230,7 @@ def execute_plugin(db, plugin):
                 if len(columns) == 9:
                     sqlParams.append((plugin["unique_prefix"], columns[0], columns[1], 'null', columns[2], columns[3], columns[4], columns[5], columns[6], 0, columns[7], 'null', columns[8]))
                 else:
-                    mylog('none', ['[Plugins]: Skipped invalid line in the output: ', line])
+                    mylog('none', ['[Plugins] Skipped invalid line in the output: ', line])
         else:
             mylog('debug', [f'[Plugins] The file {file_path} does not exist'])             
     
@@ -249,7 +250,7 @@ def execute_plugin(db, plugin):
             if len(row) == 9 and (row[0] in ['','null']) == False :
                 sqlParams.append((plugin["unique_prefix"], row[0], handle_empty(row[1]), 'null', row[2], row[3], row[4], handle_empty(row[5]), handle_empty(row[6]), 0, row[7], 'null', row[8]))
             else:
-                mylog('none', ['[Plugins]: Skipped invalid sql result'])
+                mylog('none', ['[Plugins] Skipped invalid sql result'])
 
 
     # check if the subprocess / SQL query failed / there was no valid output
@@ -257,7 +258,7 @@ def execute_plugin(db, plugin):
         mylog('none', ['[Plugins] No output received from the plugin ', plugin["unique_prefix"], ' - enable LOG_LEVEL=debug and check logs'])
         return  
     else: 
-        mylog('verbose', ['[Plugins]: SUCCESS, received ', len(sqlParams), ' entries'])  
+        mylog('verbose', ['[Plugins] SUCCESS, received ', len(sqlParams), ' entries'])  
 
     # process results if any
     if len(sqlParams) > 0:                
@@ -293,20 +294,27 @@ def passable_string_from_setting(globalSetting):
 
 
     noConversion = ['text', 'string', 'integer', 'boolean', 'password', 'readonly', 'integer.select', 'text.select', 'integer.checkbox'  ]
-    arrayConversion = ['text.multiselect', 'list', 'subnets'] 
+    arrayConversion = ['text.multiselect', 'list'] 
+    arrayConversionBase64 = ['subnets'] 
     jsonConversion = ['.template'] 
 
+    mylog('debug', f'[Plugins] setTyp: {setTyp}')
+
     if setTyp in noConversion:
         return setVal
 
     if setTyp in arrayConversion:
         return flatten_array(setVal)
 
+    if setTyp in arrayConversionBase64:
+        
+        return flatten_array(setVal, encodeBase64 = True)
+
     for item in jsonConversion:
         if setTyp.endswith(item):
             return json.dumps(setVal)
 
-    mylog('none', ['[Plugins]: ERROR: Parameter not converted.'])  
+    mylog('none', ['[Plugins] ERROR: Parameter not converted.'])  
 
 
 
@@ -337,33 +345,47 @@ def get_setting_value(key):
     return ''
 
 #-------------------------------------------------------------------------------
-def flatten_array(arr):
+def flatten_array(arr, encodeBase64=False):
-    
     tmp = ''
+    arrayItemStr = ''
 
+    mylog('debug', '[Plugins] Flattening the below array')
+    mylog('debug', f'[Plugins] Convert to Base64: {encodeBase64}')
     mylog('debug', arr)
 
-    for arrayItem in arr:        
+    for arrayItem in arr:
         # only one column flattening is supported
         if isinstance(arrayItem, list):            
-            arrayItem = str(arrayItem[0])
+            arrayItemStr = str(arrayItem[0]).replace("'", '')  # removing single quotes - not allowed            
+        else:
+            # is string already
+            arrayItemStr = arrayItem
 
-        tmp += arrayItem + ','
-        # tmp = tmp.replace("'","").replace(' ','') # No single quotes or empty spaces allowed
-        tmp = tmp.replace("'","") # No single quotes allowed
 
-    return tmp[:-1] # Remove last comma ','
+        tmp += f'{arrayItemStr},'
+
+    tmp = tmp[:-1]  # Remove last comma ','
+
+    mylog('debug', f'[Plugins] Flattened array: {tmp}')
+
+    if encodeBase64:
+        tmp = str(base64.b64encode(tmp.encode('ascii')))
+        mylog('debug', f'[Plugins] Flattened array (base64): {tmp}')
+
+
+    return tmp
+
 
 
 #-------------------------------------------------------------------------------
 # Replace {wildcars} with parameters
 def resolve_wildcards_arr(commandArr, params):
 
-    mylog('debug', ['[Plugins]: Pre-Resolved CMD: '] + commandArr)   
+    mylog('debug', ['[Plugins] Pre-Resolved CMD: '] + commandArr)   
 
     for param in params:
-        # mylog('debug', ['[Plugins]: key     : {', param[0], '}'])
+        # mylog('debug', ['[Plugins] key     : {', param[0], '}'])
-        # mylog('debug', ['[Plugins]: resolved: ', param[1]])
+        # mylog('debug', ['[Plugins] resolved: ', param[1]])
 
         i = 0
         
@@ -493,67 +515,78 @@ def process_plugin_events(db, plugin):
     # Perform databse table mapping if enabled for the plugin   
     if len(pluginEvents) > 0 and  "mapped_to_table" in plugin:
 
-        sqlParams = []        
+    # Initialize an empty list to store SQL parameters.
+    sqlParams = []
 
-        dbTable = plugin['mapped_to_table']
+    # Get the database table name from the 'mapped_to_table' key in the 'plugin' dictionary.
+    dbTable = plugin['mapped_to_table']
 
-        mylog('debug', ['[Plugins] Mapping objects to database table: ', dbTable])
+    # Log a debug message indicating the mapping of objects to the database table.
+    mylog('debug', ['[Plugins] Mapping objects to database table: ', dbTable])
 
-        #  collect all columns to be mapped
+    # Initialize lists to hold mapped column names, columnsStr, and valuesStr for SQL query.
-        mappedCols = []        
+    mappedCols = []
-        columnsStr = ''
+    columnsStr = ''
-        valuesStr = ''
+    valuesStr = ''
 
-        for clmn in plugin['database_column_definitions']:            
+    # Loop through the 'database_column_definitions' in the 'plugin' dictionary to collect mapped columns.
-            if 'mapped_to_column' in clmn:                
+    # Build the columnsStr and valuesStr for the SQL query.
-                mappedCols.append(clmn)
+    for clmn in plugin['database_column_definitions']:
-                columnsStr = f'{columnsStr}, "{clmn["mapped_to_column"]}"'
+        if 'mapped_to_column' in clmn:
-                valuesStr = f'{valuesStr}, ?'
+            mappedCols.append(clmn)
+            columnsStr = f'{columnsStr}, "{clmn["mapped_to_column"]}"'
+            valuesStr = f'{valuesStr}, ?'
 
-        if len(columnsStr) > 0:
+    # Remove the first ',' from columnsStr and valuesStr.
-            columnsStr = columnsStr[1:] # remove first ','
+    if len(columnsStr) > 0:
-            valuesStr = valuesStr[1:] # remove first ','
+        columnsStr = columnsStr[1:]
+        valuesStr = valuesStr[1:]
 
-        # map the column names to plugin object event values
+    # Map the column names to plugin object event values and create a list of tuples 'sqlParams'.
-        for plgEv in pluginEvents:
+    for plgEv in pluginEvents:
+        tmpList = []
 
-            tmpList = []
+        for col in mappedCols:
+            if col['column'] == 'Index':
+                tmpList.append(plgEv.index)
+            elif col['column'] == 'Plugin':
+                tmpList.append(plgEv.pluginPref)
+            elif col['column'] == 'Object_PrimaryID':
+                tmpList.append(plgEv.primaryId)
+            elif col['column'] == 'Object_SecondaryID':
+                tmpList.append(plgEv.secondaryId)
+            elif col['column'] == 'DateTimeCreated':
+                tmpList.append(plgEv.created)
+            elif col['column'] == 'DateTimeChanged':
+                tmpList.append(plgEv.changed)
+            elif col['column'] == 'Watched_Value1':
+                tmpList.append(plgEv.watched1)
+            elif col['column'] == 'Watched_Value2':
+                tmpList.append(plgEv.watched2)
+            elif col['column'] == 'Watched_Value3':
+                tmpList.append(plgEv.watched3)
+            elif col['column'] == 'Watched_Value4':
+                tmpList.append(plgEv.watched4)
+            elif col['column'] == 'UserData':
+                tmpList.append(plgEv.userData)
+            elif col['column'] == 'Extra':
+                tmpList.append(plgEv.extra)
+            elif col['column'] == 'Status':
+                tmpList.append(plgEv.status)
 
-            for col in mappedCols:
+        # Append the mapped values to the list 'sqlParams' as a tuple.
-                if col['column'] == 'Index':
+        sqlParams.append(tuple(tmpList))
-                    tmpList.append(plgEv.index)
-                elif col['column'] == 'Plugin':
-                    tmpList.append(plgEv.pluginPref)
-                elif col['column'] == 'Object_PrimaryID':
-                    tmpList.append(plgEv.primaryId)
-                elif col['column'] == 'Object_SecondaryID':
-                    tmpList.append(plgEv.secondaryId)
-                elif col['column'] == 'DateTimeCreated':
-                    tmpList.append(plgEv.created)
-                elif col['column'] == 'DateTimeChanged':
-                    tmpList.append(plgEv.changed)
-                elif col['column'] == 'Watched_Value1':
-                    tmpList.append(plgEv.watched1)
-                elif col['column'] == 'Watched_Value2':
-                    tmpList.append(plgEv.watched2)
-                elif col['column'] == 'Watched_Value3':
-                    tmpList.append(plgEv.watched3)
-                elif col['column'] == 'Watched_Value4':
-                    tmpList.append(plgEv.watched4)
-                elif col['column'] == 'UserData':
-                    tmpList.append(plgEv.userData)
-                elif col['column'] == 'Extra':
-                    tmpList.append(plgEv.extra)
-                elif col['column'] == 'Status':
-                    tmpList.append(plgEv.status)               
-                
-            sqlParams.append(tuple(tmpList))
 
-        q = f'INSERT into {dbTable} ({columnsStr}) VALUES ({valuesStr})'
+    # Generate the SQL INSERT query using the collected information.
+    q = f'INSERT into {dbTable} ({columnsStr}) VALUES ({valuesStr})'
 
-        mylog('debug', ['[Plugins] SQL query for mapping: ', q ])
+    # Log a debug message showing the generated SQL query for mapping.
+    mylog('debug', ['[Plugins] SQL query for mapping: ', q])
+
+    # Execute the SQL query using 'sql.executemany()' and the 'sqlParams' list of tuples.
+    # This will insert multiple rows into the database in one go.
+    sql.executemany(q, sqlParams)
 
-        sql.executemany (q, sqlParams) 
 
     db.commitDB()