Online history work 👷

This commit is contained in:
Jokob-sk
2023-11-05 09:40:04 +11:00
parent f5d8a9fc8c
commit 89fb5c9b3b
22 changed files with 72 additions and 62 deletions

View File

@@ -7,15 +7,15 @@ echo "---------------------------------------------------------"
INSTALL_DIR=/home/pi # Specify the installation directory here
# # if custom variables not set we do not need to do anything
# if [ -n "${TZ}" ]; then
# FILECONF=$INSTALL_DIR/pialert/config/pialert.conf
# if [ -f "$FILECONF" ]; then
# sed -ie "s|Europe/Berlin|${TZ}|g" $INSTALL_DIR/pialert/config/pialert.conf
# else
# sed -ie "s|Europe/Berlin|${TZ}|g" $INSTALL_DIR/pialert/back/pialert.conf_bak
# fi
# fi
# if custom variables not set we do not need to do anything
if [ -n "${TZ}" ]; then
FILECONF=$INSTALL_DIR/pialert/config/pialert.conf
if [ -f "$FILECONF" ]; then
sed -ie "s|Europe/Berlin|${TZ}|g" $INSTALL_DIR/pialert/config/pialert.conf
else
sed -ie "s|Europe/Berlin|${TZ}|g" $INSTALL_DIR/pialert/back/pialert.conf_bak
fi
fi
# Check if script is run as root
if [[ $EUID -ne 0 ]]; then

View File

@@ -29,6 +29,7 @@ USER_HOME=${array[4]}
echo USER_ID : ${USER_ID};
echo USER_GID : ${USER_GID};
echo USER_HOME: ${USER_HOME};
echo TZ : ${TZ};
sed -i -e "s/^${USER}:\([^:]*\):[0-9]*:[0-9]*/${USER}:\1:${USER_ID}:${USER_GID}/" /etc/passwd
sed -i -e "s/^${USER}:\([^:]*\):[0-9]*/${USER}:\1:${USER_GID}/" /etc/group

View File

@@ -30,7 +30,7 @@ def main():
# Check if basic config settings supplied
if check_config() == False:
mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
mylog('none', [f'[{pluginName}] ⚠ ERROR: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
return
# Create a database connection

View File

@@ -38,7 +38,7 @@ def main():
# Check if basic config settings supplied
if check_config() == False:
mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
mylog('none', [f'[{pluginName}] ⚠ ERROR: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
return
# Create a database connection
@@ -82,7 +82,7 @@ def check_config ():
report_from = get_setting_value("SMTP_REPORT_FROM")
if server == '' or report_from == '' or report_to == '':
mylog('none', ['[Email Check Config] Error: Email service not set up correctly. Check your pialert.conf SMTP_*, SMTP_REPORT_FROM and SMTP_REPORT_TO variables.'])
mylog('none', ['[Email Check Config] ⚠ ERROR: Email service not set up correctly. Check your pialert.conf SMTP_*, SMTP_REPORT_FROM and SMTP_REPORT_TO variables.'])
return False
else:
return True

View File

@@ -51,7 +51,7 @@ def main():
# Check if basic config settings supplied
if check_config() == False:
mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
mylog('none', [f'[{pluginName}] ⚠ ERROR: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
return
# Create a database connection
@@ -70,7 +70,7 @@ def main():
#-------------------------------------------------------------------------------
def check_config():
if get_setting_value('MQTT_BROKER') == '' or get_setting_value('MQTT_PORT') == '' or get_setting_value('MQTT_USER') == '' or get_setting_value('MQTT_PASSWORD') == '':
mylog('none', ['[Check Config] Error: MQTT service not set up correctly. Check your pialert.conf MQTT_* variables.'])
mylog('none', ['[Check Config] ⚠ ERROR: MQTT service not set up correctly. Check your pialert.conf MQTT_* variables.'])
return False
else:
return True

View File

@@ -33,7 +33,7 @@ def main():
# Check if basic config settings supplied
if check_config() == False:
mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
mylog('none', [f'[{pluginName}] ⚠ ERROR: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
return
# Create a database connection
@@ -118,7 +118,7 @@ def send(html, text):
response_text = json.dumps(response.text)
except requests.exceptions.RequestException as e:
mylog('none', [f'[{pluginName}] Error: ', e])
mylog('none', [f'[{pluginName}] ⚠ ERROR: ', e])
response_text = e

View File

@@ -32,7 +32,7 @@ def main():
# Check if basic config settings supplied
if check_config() == False:
mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
mylog('none', [f'[{pluginName}] ⚠ ERROR: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
return
# Create a database connection
@@ -107,7 +107,7 @@ def send(text):
response_text = json.dumps(response.text)
except requests.exceptions.RequestException as e:
mylog('none', [f'[{pluginName}] Error: ', e])
mylog('none', [f'[{pluginName}] ⚠ ERROR: ', e])
response_text = e

View File

@@ -36,7 +36,7 @@ def main():
# Check if basic config settings supplied
if check_config() == False:
mylog('none', [f'[{pluginName}] Error: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
mylog('none', [f'[{pluginName}] ⚠ ERROR: Publisher notification gateway not set up correctly. Check your pialert.conf {pluginName}_* variables.'])
return
# Create a database connection
@@ -177,7 +177,7 @@ def send (text_data, html_data, json_data):
except subprocess.CalledProcessError as e:
# An error occurred, handle it
mylog('none', [f'[{pluginName}] Error: ', e.output])
mylog('none', [f'[{pluginName}] ⚠ ERROR: ', e.output])
response_stderr = e.output

View File

@@ -109,7 +109,7 @@ def get_dynamic_DNS_IP (DDNS_DOMAIN):
mylog('none', [f'[{pluginName}] DIG output :', dig_output])
except subprocess.CalledProcessError as e:
# An error occured, handle it
mylog('none', [f'[{pluginName}] ERROR - ', e.output])
mylog('none', [f'[{pluginName}] ERROR - ', e.output])
dig_output = '' # probably no internet
# Check result is an IP
@@ -135,7 +135,7 @@ def set_dynamic_DNS_IP (DDNS_UPDATE_URL, DDNS_USER, DDNS_PASSWORD, DDNS_DOMAIN):
universal_newlines=True)
except subprocess.CalledProcessError as e:
# An error occured, handle it
mylog('none', [f'[{pluginName}] ERROR - ',e.output])
mylog('none', [f'[{pluginName}] ERROR - ',e.output])
curl_output = ""
return curl_output

View File

@@ -46,7 +46,7 @@ def get_entries(path, plugin_objects):
# Check if the path exists
if not os.path.exists(path):
mylog('none', [f'[{pluginName}] Error: "{path}" does not exist.'])
mylog('none', [f'[{pluginName}] ⚠ ERROR: "{path}" does not exist.'])
else:
# Detect file encoding
with open(path, 'rb') as f:

View File

@@ -119,7 +119,7 @@ def performNmapScan(deviceIPs, deviceMACs, timeoutSec, args):
except subprocess.CalledProcessError as e:
# An error occured, handle it
mylog('none', ["[NMAP Scan] " ,e.output])
mylog('none', ["[NMAP Scan] Error - Nmap Scan - check logs", progress])
mylog('none', ["[NMAP Scan] ⚠ ERROR - Nmap Scan - check logs", progress])
except subprocess.TimeoutExpired as timeErr:
mylog('verbose', ['[NMAP Scan] Nmap TIMEOUT - the process forcefully terminated as timeout reached for ', ip, progress])

View File

@@ -150,7 +150,7 @@ def execute_pholus_on_interface(interface, timeoutSec, mask):
except subprocess.CalledProcessError as e:
# An error occured, handle it
mylog('none', ['[PHOLUS]', e.output])
mylog('none', ["[PHOLUS] Error - Pholus Scan - check logs"])
mylog('none', ["[PHOLUS] ⚠ ERROR - Pholus Scan - check logs"])
except subprocess.TimeoutExpired as timeErr:
mylog('none', ['[PHOLUS] Pholus TIMEOUT - the process forcefully terminated as timeout reached'])

View File

@@ -188,7 +188,6 @@ def main ():
mylog('verbose', ['[MAIN] Process: Wait'])
else:
# do something
conf.cycle = ""
mylog('verbose', ['[MAIN] waiting to start next loop'])
#loop

View File

@@ -87,5 +87,5 @@ class api_endpoint_class:
# update hash
apiEndpoints[changedIndex].hash = self.hash
else:
mylog('minimal', [f'[API] ERROR Updating {self.fileName}'])
mylog('minimal', [f'[API] ERROR Updating {self.fileName}'])

View File

@@ -173,13 +173,13 @@ def getPluginObject(**kwargs):
return item
mylog('debug', [f'[{module_name}] Error - Object not found - GUID:{GUID} | Plugin:{Plugin} | MAC:{MAC} | IP:{IP} | PrimaryID:{PrimaryID} | SecondaryID:{SecondaryID} | ForeignKey:{ForeignKey} | Index:{Index} | RowID:{RowID} '])
mylog('debug', [f'[{module_name}] ⚠ ERROR - Object not found - GUID:{GUID} | Plugin:{Plugin} | MAC:{MAC} | IP:{IP} | PrimaryID:{PrimaryID} | SecondaryID:{SecondaryID} | ForeignKey:{ForeignKey} | Index:{Index} | RowID:{RowID} '])
return None
except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
# Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format
mylog('none', [f'[{module_name}] Error - JSONDecodeError or FileNotFoundError for file {plugins_objects}'])
mylog('none', [f'[{module_name}] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {plugins_objects}'])
return None

View File

@@ -172,7 +172,7 @@ def create_new_devices (db):
FROM CurrentScan"""
# mylog('debug',f'[New Devices] Create devices SQL: {sqlQuery}')
mylog('debug',f'[New Devices] Create devices SQL: {sqlQuery}')
sql.execute (sqlQuery, (startTime, startTime) )
@@ -359,6 +359,6 @@ def query_MAC_vendor (pMAC):
return -1 # MAC address not found in the database
except FileNotFoundError:
mylog('none', [f"[Vendor Check] Error: Vendors file {vendorsPath} not found."])
mylog('none', [f"[Vendor Check] ⚠ ERROR: Vendors file {vendorsPath} not found."])
return -1

View File

@@ -23,12 +23,16 @@ from logger import mylog, logResult
#-------------------------------------------------------------------------------
# Get the current time in the current TimeZone
def timeNowTZ():
if isinstance(conf.TIMEZONE, str):
tz = pytz.timezone(conf.TIMEZONE)
if conf.tz:
return datetime.datetime.now(conf.tz).replace(microsecond=0)
else:
tz = conf.TIMEZONE
return datetime.datetime.now().replace(microsecond=0)
# if isinstance(conf.TIMEZONE, str):
# tz = pytz.timezone(conf.TIMEZONE)
# else:
# tz = conf.TIMEZONE
return datetime.datetime.now(tz).replace(microsecond=0)
# return datetime.datetime.now(tz).replace(microsecond=0)
def timeNow():
return datetime.datetime.now().replace(microsecond=0)
@@ -173,7 +177,7 @@ def initialiseFile(pathToCheck, defaultFile):
stdout, stderr = p.communicate()
if str(os.access(pathToCheck, os.R_OK)) == "False":
mylog('none', ["[Setup] Error copying ("+defaultFile+") to ("+pathToCheck+"). Make sure the app has Read & Write access to the parent directory."])
mylog('none', ["[Setup] ⚠ ERROR copying ("+defaultFile+") to ("+pathToCheck+"). Make sure the app has Read & Write access to the parent directory."])
else:
mylog('none', ["[Setup] ("+defaultFile+") copied over successfully to ("+pathToCheck+")."])
@@ -182,7 +186,7 @@ def initialiseFile(pathToCheck, defaultFile):
except subprocess.CalledProcessError as e:
# An error occured, handle it
mylog('none', ["[Setup] Error copying ("+defaultFile+"). Make sure the app has Read & Write access to " + pathToCheck])
mylog('none', ["[Setup] ⚠ ERROR copying ("+defaultFile+"). Make sure the app has Read & Write access to " + pathToCheck])
mylog('none', [e.output])
#-------------------------------------------------------------------------------
@@ -255,13 +259,13 @@ def get_setting(key):
if item.get("Code_Name") == key:
return item
mylog('debug', [f'[Settings] Error - setting_missing - Setting not found for key: {key} in file {settingsFile}'])
mylog('debug', [f'[Settings] ⚠ ERROR - setting_missing - Setting not found for key: {key} in file {settingsFile}'])
return None
except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
# Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format
mylog('none', [f'[Settings] Error - JSONDecodeError or FileNotFoundError for file {settingsFile}'])
mylog('none', [f'[Settings] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {settingsFile}'])
return None
@@ -290,11 +294,15 @@ def get_setting_value(key):
value = str(set_value)
elif set_type in ['boolean', 'integer.checkbox']:
value = True
if set_value in ['false', 'False', 'FALSE', 0]:
value = False
if isinstance(set_value, str) and set_value.lower() in ['true', '1']:
value = True
elif isinstance(set_value, int) and set_value == 1:
value = True
elif isinstance(set_value, bool):
value = set_value
elif set_type in ['integer.select', 'integer']:
value = int(set_value)
elif set_type in ['text.multiselect', 'list', 'subnets']:
@@ -304,8 +312,8 @@ def get_setting_value(key):
# Assuming set_value is a JSON object in this case
value = json.loads(set_value)
else:
mylog('none', [f'[SETTINGS] ERROR - set_type not handled:{set_type}'])
mylog('none', [f'[SETTINGS] ERROR - setting json:{json.dumps(setting)}'])
mylog('none', [f'[SETTINGS] ERROR - set_type not handled:{set_type}'])
mylog('none', [f'[SETTINGS] ERROR - setting json:{json.dumps(setting)}'])
return value
@@ -378,7 +386,7 @@ def resolve_device_name_dig (pMAC, pIP):
except subprocess.CalledProcessError as e:
# An error occured, handle it
mylog('none', ['[resolve_device_name_dig] ERROR: ', e.output])
mylog('none', ['[resolve_device_name_dig] ERROR: ', e.output])
# newName = "Error - check logs"
return nameNotFound
@@ -693,7 +701,7 @@ def checkNewVersion():
text = url.text
data = json.loads(text)
except requests.exceptions.ConnectionError as e:
mylog('minimal', ["[Version check] Error: Couldn't check for new release."])
mylog('minimal', ["[Version check] ⚠ ERROR: Couldn't check for new release."])
data = ""
# make sure we received a valid response and not an API rate limit exceeded message

View File

@@ -124,7 +124,6 @@ def importConfigs (db):
# TODO cleanup later ----------------------------------------------------------------------------------
# init all time values as we have timezone - all this shoudl be moved into plugin/plugin settings
conf.time_started = datetime.datetime.now(conf.tz)
conf.cycle = ""
conf.plugins_once_run = False
# timestamps of last execution times

View File

@@ -20,10 +20,13 @@ def timeNowTZ():
#-------------------------------------------------------------------------------
# More verbose as the numbers go up
debugLevels = [
('none', 0), ('minimal', 1), ('verbose', 2), ('debug', 3)
]
currentLevel = 0
def mylog(requestedDebugLevel, n):
setLvl = 0

View File

@@ -53,7 +53,7 @@ class plugin_param:
if setTyp.endswith(item):
return json.dumps(setVal)
else:
mylog('none', ['[Plugins] ERROR: Parameter not converted.'])
mylog('none', ['[Plugins] ERROR: Parameter not converted.'])
# Get SQL result
@@ -213,7 +213,7 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
except subprocess.CalledProcessError as e:
# An error occured, handle it
mylog('none', [e.output])
mylog('none', ['[Plugins] Error - enable LOG_LEVEL=debug and check logs'])
mylog('none', ['[Plugins] ⚠ ERROR - enable LOG_LEVEL=debug and check logs'])
except subprocess.TimeoutExpired as timeErr:
mylog('none', ['[Plugins] TIMEOUT - the process forcefully terminated as timeout reached'])
@@ -317,7 +317,7 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
# handle missing "function":"DB_PATH" setting
if set == None:
mylog('none', ['[Plugins] Error: DB_PATH setting for plugin type sqlite-db-query missing.'])
mylog('none', ['[Plugins] ⚠ ERROR: DB_PATH setting for plugin type sqlite-db-query missing.'])
return pluginsState
fullSqlitePath = set["value"]
@@ -328,8 +328,8 @@ def execute_plugin(db, plugin, pluginsState = plugins_state() ):
sql.execute ("ATTACH DATABASE '"+ fullSqlitePath +"' AS EXTERNAL_"+plugin["unique_prefix"])
arr = db.get_sql_array (q)
except sqlite3.Error as e:
mylog('none',[f'[Plugins] Error: DB_PATH setting ({fullSqlitePath}) for plugin {plugin["unique_prefix"]}. Did you mount it correctly?'])
mylog('none',[f'[Plugins] Error: ATTACH DATABASE failed with SQL ERROR: ', e])
mylog('none',[f'[Plugins] ⚠ ERROR: DB_PATH setting ({fullSqlitePath}) for plugin {plugin["unique_prefix"]}. Did you mount it correctly?'])
mylog('none',[f'[Plugins] ⚠ ERROR: ATTACH DATABASE failed with SQL ERROR: ', e])
return pluginsState
for row in arr:
@@ -580,7 +580,7 @@ def process_plugin_events(db, plugin, pluginsState, plugEventsArr):
except Exception as e:
# Rollback the transaction in case of an error
conn.rollback()
mylog('none', ['[Plugins] Error: ', e])
mylog('none', ['[Plugins] ⚠ ERROR: ', e])
raise e
# Perform database table mapping if enabled for the plugin

View File

@@ -114,7 +114,7 @@ def list_to_csv(arr):
return tmp
else:
mylog('none', f'[{module_name}] ERROR Could not convert array: {arr}')
mylog('none', f'[{module_name}] ERROR Could not convert array: {arr}')
@@ -218,13 +218,13 @@ def getPluginObject(keyValues):
if all_match:
return item
mylog('verbose', [f'[{module_name}] Error - Object not found {json.dumps(keyValues)} '])
mylog('verbose', [f'[{module_name}] ⚠ ERROR - Object not found {json.dumps(keyValues)} '])
return {}
except (FileNotFoundError, json.JSONDecodeError, ValueError) as e:
# Handle the case when the file is not found, JSON decoding fails, or data is not in the expected format
mylog('verbose', [f'[{module_name}] Error - JSONDecodeError or FileNotFoundError for file {plugins_objects}'])
mylog('verbose', [f'[{module_name}] ⚠ ERROR - JSONDecodeError or FileNotFoundError for file {plugins_objects}'])
return {}