pholus 2

Dockerfile

@@ -8,7 +8,7 @@ ENV USER=pi USER_ID=1000 USER_GID=1000 TZ=Europe/London PORT=20211
 
 RUN apt-get update \
     && apt-get install --no-install-recommends tini ca-certificates curl libwww-perl arp-scan perl apt-utils cron sudo nginx-light php php-cgi php-fpm php-sqlite3 php-curl sqlite3 dnsutils net-tools python3 iproute2 nmap python3-pip zip -y \
-    && pip3 install requests paho-mqtt scapy \
+    && pip3 install requests paho-mqtt scapy cron-converter pytz \
     # && pip3 install requests paho-mqtt ssdpy upnpclient \
     && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \
     && apt-get clean autoclean \

back/pialert.py

@@ -24,6 +24,8 @@ import re
 import time
 import datetime
 from datetime import timedelta
+# from datetime import datetime
+# from datetime import date
 import sqlite3
 import socket
 import io
@@ -35,6 +37,9 @@ from base64 import b64encode
 from paho.mqtt import client as mqtt_client
 import threading
 from pathlib import Path
+from cron_converter import Cron
+from pytz import timezone
+
 # from ssdpy import SSDPClient
 # import upnpclient
 
@@ -52,9 +57,13 @@ fullDbPath   = pialertPath + dbPath
 STOPARPSCAN = pialertPath + "/db/setting_stoparpscan"
 
 time_started = datetime.datetime.now()
+cron_instance = Cron()
 log_timestamp = time_started
+lastTimeImported = 0
 sql_connection = None
 
+next_schedule_timestamp = 0
+
 
 #-------------------------------------------------------------------------------
 def timeNow():
@@ -154,7 +163,6 @@ initialiseFile(fullDbPath, "/home/pi/pialert/back/pialert.db_bak")
 #===============================================================================
 
 vendorsDB              = '/usr/share/arp-scan/ieee-oui.txt'
-
 piholeDB               = '/etc/pihole/pihole-FTL.db'
 piholeDhcpleases       = '/etc/pihole/dhcp.leases'
 
@@ -171,6 +179,8 @@ SCAN_CYCLE_MINUTES      = 5
 
 SCAN_SUBNETS            = ['192.168.1.0/24 --interface=eth1', '192.168.1.0/24 --interface=eth0']
 
+DAYS_TO_KEEP_EVENTS     = 90
+
 # EMAIL settings
 # ----------------------
 SMTP_SERVER             = ''
@@ -230,18 +240,22 @@ DDNS_USER               = 'dynu_user'
 DDNS_PASSWORD           = 'A0000000B0000000C0000000D0000000'
 DDNS_UPDATE_URL         = 'https://api.dynu.com/nic/update?'
 
-# Pholus settings
-# ----------------------
-PHOLUS_ACTIVE           = False                         
-PHOLUS_TIMEOUT          = 60   
-
 # PIHOLE settings
 # ----------------------
 PIHOLE_ACTIVE           = False                         
 DHCP_ACTIVE             = False                         
 
-# keep 90 days of network activity if not specified how many days to keep
+
-DAYS_TO_KEEP_EVENTS     = 90
+# Pholus settings
+# ----------------------
+PHOLUS_ACTIVE           = False
+PHOLUS_TIMEOUT          = 60   
+PHOLUS_FORCE            = False
+PHOLUS_DAYS_DATA        = 7
+
+PHOLUS_RUN              = 'none'
+PHOLUS_RUN_SCHD         = '0 4 * * *'
+PHOLUS_RUN_TIMEOUT      = 600
 
 
 #===============================================================================
@@ -300,6 +314,7 @@ def check_config_dict(key, default, config):
 def importConfig (): 
 
     # Specify globals so they can be overwritten with the new config
+    global lastTimeImported
     # General
     global SCAN_SUBNETS, PRINT_LOG, TIMEZONE, PIALERT_WEB_PROTECTION, PIALERT_WEB_PASSWORD, INCLUDED_SECTIONS, SCAN_CYCLE_MINUTES, DAYS_TO_KEEP_EVENTS, REPORT_DASHBOARD_URL
     # Email
@@ -319,14 +334,22 @@ def importConfig ():
     # PiHole
     global PIHOLE_ACTIVE, DHCP_ACTIVE
     # Pholus
-    global PHOLUS_ACTIVE, PHOLUS_TIMEOUT  
+    global PHOLUS_ACTIVE, PHOLUS_TIMEOUT, PHOLUS_FORCE, PHOLUS_DAYS_DATA, PHOLUS_RUN, PHOLUS_RUN_SCHD, PHOLUS_RUN_TIMEOUT
+
+    # get config file
+    config_file = Path(fullConfPath)
+
+    # Skip import if last time of import is NEWER than file age 
+    if (os.path.getmtime(config_file) < lastTimeImported) :
+        return
     
     # load the variables from  pialert.conf
-    config_file = Path(fullConfPath)
     code = compile(config_file.read_text(), config_file.name, "exec")
     config_dict = {}
     exec(code, {"__builtins__": {}}, config_dict)
 
+    # Import setting if found in the dictionary
+    # General
     SCAN_SUBNETS = check_config_dict('SCAN_SUBNETS', SCAN_SUBNETS , config_dict)
     PRINT_LOG = check_config_dict('PRINT_LOG', PRINT_LOG , config_dict)
     TIMEZONE = check_config_dict('TIMEZONE', TIMEZONE , config_dict)
@@ -393,6 +416,12 @@ def importConfig ():
     # PHOLUS
     PHOLUS_ACTIVE = check_config_dict('PHOLUS_ACTIVE', PHOLUS_ACTIVE , config_dict)
     PHOLUS_TIMEOUT = check_config_dict('PHOLUS_TIMEOUT', PHOLUS_TIMEOUT , config_dict)
+    PHOLUS_FORCE = check_config_dict('PHOLUS_FORCE', PHOLUS_FORCE , config_dict)
+    PHOLUS_DAYS_DATA = check_config_dict('PHOLUS_DAYS_DATA', PHOLUS_DAYS_DATA , config_dict)
+    PHOLUS_RUN = check_config_dict('PHOLUS_RUN', PHOLUS_RUN , config_dict)
+    PHOLUS_RUN_TIMEOUT = check_config_dict('PHOLUS_RUN_TIMEOUT', PHOLUS_RUN_TIMEOUT , config_dict)   
+    PHOLUS_RUN_SCHD = check_config_dict('PHOLUS_RUN_SCHD', PHOLUS_RUN_SCHD , config_dict)
+ 
 
     openDB()    
 
@@ -466,7 +495,12 @@ def importConfig ():
 
         # Pholus
         ('PHOLUS_ACTIVE', 'Enable Pholus scans', '',  'boolean', '', '' , str(PHOLUS_ACTIVE) , 'Pholus'),
-        ('PHOLUS_TIMEOUT', 'Pholus timeout', '',  'integer', '', '' , str(PHOLUS_TIMEOUT) , 'Pholus')
+        ('PHOLUS_TIMEOUT', 'Pholus timeout', '',  'integer', '', '' , str(PHOLUS_TIMEOUT) , 'Pholus'),
+        ('PHOLUS_FORCE', 'Pholus force check', '',  'boolean', '', '' , str(PHOLUS_FORCE) , 'Pholus'),
+        ('PHOLUS_DAYS_DATA', 'Pholus keep days', '',  'integer', '', '' , str(PHOLUS_DAYS_DATA) , 'Pholus'),
+        ('PHOLUS_RUN', 'Pholus enable schedule', '',  'selecttext', "['none', 'once', 'schedule']", '' , str(PHOLUS_RUN) , 'Pholus'),
+        ('PHOLUS_RUN_TIMEOUT', 'Pholus timeout schedule', '',  'integer', '', '' , str(PHOLUS_RUN_TIMEOUT) , 'Pholus'),
+        ('PHOLUS_RUN_SCHD', 'Pholus schedule', '',  'text', '', '' , str(PHOLUS_RUN_SCHD) , 'Pholus')        
 
     ]
     # Insert into DB    
@@ -475,7 +509,23 @@ def importConfig ():
     sql.executemany ("""INSERT INTO Settings ("Code_Name", "Display_Name", "Description", "Type", "Options",
          "RegEx", "Value", "Group" ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", settings)
 
+
+    # Used to determine the next import
+    lastTimeImported = time.time()
+    
     closeDB()
+
+    # Update scheduler
+    global schedule, tz, last_next_pholus_schedule, last_next_pholus_schedule_used
+
+    tz       = timezone(TIMEZONE)    
+    cron     = Cron(PHOLUS_RUN_SCHD)
+    schedule = cron.schedule(start_date=datetime.datetime.now(tz))
+
+    last_next_pholus_schedule = schedule.next()
+    last_next_pholus_schedule_used = False
+
+    file_print('[', timeNow(), '] Config: Imported new config')  
 #-------------------------------------------------------------------------------
 
 #===============================================================================
@@ -487,30 +537,30 @@ def importConfig ():
 #===============================================================================
 cycle = ""
 check_report = [1, "internet_IP", "update_vendors_silent"]
-mqtt_thread_up = False
+last_pholus_scheduled_run = 0
 
 # timestamps of last execution times
 startTime = time_started
-now_minus_24h = time_started - timedelta(hours = 24)
+now_minus_24h = time_started - datetime.timedelta(hours = 24)
 
 last_network_scan = now_minus_24h
 last_internet_IP_scan = now_minus_24h
 last_run = now_minus_24h
 last_cleanup = now_minus_24h
-last_update_vendors = time_started - timedelta(days = 6) # update vendors 24h after first run and than once a week
+last_update_vendors = time_started - datetime.timedelta(days = 6) # update vendors 24h after first run and than once a week
 
 def main ():
     # Initialize global variables
-    global time_started, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors, mqtt_thread_up
+    global time_started, cycle, last_network_scan, last_internet_IP_scan, last_run, last_cleanup, last_update_vendors, last_pholus_scheduled_run
     # second set of global variables
     global startTime, log_timestamp, sql_connection, sql
 
-
     # DB
     sql_connection = None
     sql            = None
 
-    # # create log files
+    # # create log files > I don't think this is necessary (e.g. the path was incorrect 
+    # # (missing / at the beginning of teh file name) and there were no issues reported)
     # write_file(logPath + 'IP_changes.log', '')
     # write_file(logPath + 'stdout.log', '')
     # write_file(logPath + 'stderr.log', '')
@@ -529,7 +579,7 @@ def main ():
         importConfig() 
 
         # proceed if 1 minute passed
-        if last_run + timedelta(minutes=1) < time_started :
+        if last_run + datetime.timedelta(minutes=1) < time_started :
 
              # last time any scan or maintennace/Upkeep was run
             last_run = time_started
@@ -545,18 +595,37 @@ def main ():
             startTime = startTime.replace (microsecond=0)      
 
             # determine run/scan type based on passed time
-            if last_internet_IP_scan + timedelta(minutes=3) < time_started:
+            if last_internet_IP_scan + datetime.timedelta(minutes=3) < time_started:
                 cycle = 'internet_IP'                
                 last_internet_IP_scan = time_started
                 reporting = check_internet_IP()
 
             # Update vendors once a week
-            if last_update_vendors + timedelta(days = 7) < time_started:
+            if last_update_vendors + datetime.timedelta(days = 7) < time_started:
                 last_update_vendors = time_started
                 cycle = 'update_vendors'
                 update_devices_MAC_vendors()
 
-            if last_network_scan + timedelta(minutes=SCAN_CYCLE_MINUTES) < time_started and os.path.exists(STOPARPSCAN) == False:
+            # Execute Pholus scheduled scan if enabled and run conditions fulfilled
+            if PHOLUS_RUN == "schedule" or PHOLUS_RUN == "once":
+
+                runPholus = False
+
+                # run once after application starts
+                if PHOLUS_RUN == "once" and last_pholus_scheduled_run == 0:
+                    runPholus = True
+
+                # run if overdue scheduled time
+                if (not runPholus) and (PHOLUS_RUN == "schedule"):
+                    # cron_instance.from_string(PHOLUS_RUN_SCHD)
+                    runPholus = runSchedule()
+
+                if runPholus:
+                    last_pholus_scheduled_run = datetime.datetime.now(tz).replace(microsecond=0)
+                    performPholusScan()
+
+            # Perform an arp-scan if not disable with a file
+            if last_network_scan + datetime.timedelta(minutes=SCAN_CYCLE_MINUTES) < time_started and os.path.exists(STOPARPSCAN) == False:
                 last_network_scan = time_started
                 cycle = 1 # network scan
                 scan_network() 
@@ -566,7 +635,7 @@ def main ():
                 email_reporting()
 
             # clean up the DB once a day
-            if last_cleanup + timedelta(hours = 24) < time_started:
+            if last_cleanup + datetime.timedelta(hours = 24) < time_started:
                 last_cleanup = time_started
                 cycle = 'cleanup'  
                 cleanup_database()   
@@ -589,8 +658,8 @@ def main ():
             # do something
             cycle = ""           
 
-        #loop  - recursion    
+        #loop     
-        time.sleep(20) # wait for N seconds      
+        time.sleep(5) # wait for N seconds      
 
     
 #===============================================================================
@@ -784,8 +853,8 @@ def cleanup_database ():
     sql.execute ("DELETE FROM Events WHERE eve_DateTime <= date('now', '-"+str(DAYS_TO_KEEP_EVENTS)+" day')")
 
     # Cleanup Pholus_Scan
-    file_print('    Upkeep Pholus_Scan, delete all older than 7 days')
+    file_print('    Upkeep Pholus_Scan, delete all older than ' + str(PHOLUS_DAYS_DATA) + ' days')
-    sql.execute ("DELETE FROM Pholus_Scan WHERE Time <= date('now', '-7 day')")
+    sql.execute ("DELETE FROM Pholus_Scan WHERE Time <= date('now', '-"+ str(PHOLUS_DAYS_DATA) +" day')") # improvement possibility: keep at least N per mac
 
     # Shrink DB
     file_print('    Shrink Database')
@@ -1580,27 +1649,8 @@ def update_devices_names ():
     unknownDevices = sql.fetchall() 
 
     # perform Pholus scan if (unknown) devices found
-    if len(unknownDevices) > 0 and PHOLUS_ACTIVE:
+    if PHOLUS_ACTIVE and (len(unknownDevices) > 0 or PHOLUS_FORCE):        
-
+        performPholusScan()
-        subnetList = []
-
-        # handle old strin setting  
-        if type(SCAN_SUBNETS) is not list:                      
-            subnetList.append(SCAN_SUBNETS)
-        else:
-            subnetList = SCAN_SUBNETS
-
-        # scan every interface
-        for subnet in subnetList:
-
-            temp = subnet.strip().split()
-
-            mask = temp[0]
-            interface = temp[1].split('=')[1]
-
-            file_print(">>> Pholus scan on: ", interface,mask)
-
-            performPholusScan(interface, mask)
 
     # get names from Pholus scan 
     sql.execute ('SELECT * FROM Pholus_Scan where "MAC" in (select "dev_MAC" from Devices where "dev_Name" IN ("(unknown)","")) and "Record_Type"="Answer"')        
@@ -1635,43 +1685,60 @@ def update_devices_names ():
         # file_print(sql.rowcount)
 
 #-------------------------------------------------------------------------------
-def performPholusScan (interface, mask):
+def performPholusScan ():
-    updateState("Scan: Pholus")
-    file_print('[', timeNow(), '] Scan: Pholus')  
 
-    pholus_args = ['python3', '/home/pi/pialert/pholus/pholus3.py', interface, "-rdns_scanning", mask, "-stimeout", str(PHOLUS_TIMEOUT)]
+    subnetList = []
-
-    # Execute command
-    try:
-        # try runnning a subprocess
-        output = subprocess.check_output (pholus_args, universal_newlines=True)
-    except subprocess.CalledProcessError as e:
-        # An error occured, handle it
-        file_print(e.output)
-        file_print("Error - PholusScan - check logs")
-        output = ""
-
-    if output != "":
-        file_print('[', timeNow(), '] Scan: Pholus SUCCESS')
-        write_file (logPath + '/pialert_pholus_old.log', output)
-        for line in output.split("\n"):
-            append_line_to_file (logPath + '/pialert_pholus.log', line +'\n')
-        
-       
-
-        params = []
-
-        for line in output.split("\n"):
-            columns = line.split("|")
-            if len(columns) == 4:
-                params.append(( interface + " " + mask, timeNow() , columns[0].replace(" ", ""), columns[1].replace(" ", ""), columns[2].replace(" ", ""), columns[3], ''))
-
-        if len(params) > 0:
-            openDB ()
-            sql.executemany ("""INSERT INTO Pholus_Scan ("Info", "Time", "MAC", "IP_v4_or_v6", "Record_Type", "Value", "Extra") VALUES (?, ?, ?, ?, ?, ?, ?)""", params) 
 
+    # handle old string setting  
+    if type(SCAN_SUBNETS) is not list:                      
+        subnetList.append(SCAN_SUBNETS)
     else:
-        file_print('[', timeNow(), '] Scan: Pholus FAIL - check logs') 
+        subnetList = SCAN_SUBNETS
+
+    # scan every interface
+    for subnet in subnetList:
+
+        temp = subnet.strip().split()
+
+        mask = temp[0]
+        interface = temp[1].split('=')[1]
+
+        file_print("        Pholus scan on interface: ", interface, " mask: " , mask)
+
+        updateState("Scan: Pholus")        
+        file_print('[', timeNow(), '] Scan: Pholus')  
+        
+        pholus_args = ['python3', '/home/pi/pialert/pholus/pholus3.py', interface, "-rdns_scanning", mask, "-stimeout", str(PHOLUS_TIMEOUT)]
+
+        # Execute command
+        try:
+            # try runnning a subprocess
+            output = subprocess.check_output (pholus_args, universal_newlines=True)
+        except subprocess.CalledProcessError as e:
+            # An error occured, handle it
+            file_print(e.output)
+            file_print("Error - PholusScan - check logs")
+            output = ""
+
+        if output != "":
+            file_print('[', timeNow(), '] Scan: Pholus SUCCESS')
+            write_file (logPath + '/pialert_pholus_old.log', output)
+            for line in output.split("\n"):
+                append_line_to_file (logPath + '/pialert_pholus.log', line +'\n')        
+
+            params = []
+
+            for line in output.split("\n"):
+                columns = line.split("|")
+                if len(columns) == 4:
+                    params.append(( interface + " " + mask, timeNow() , columns[0].replace(" ", ""), columns[1].replace(" ", ""), columns[2].replace(" ", ""), columns[3], ''))
+
+            if len(params) > 0:
+                openDB ()
+                sql.executemany ("""INSERT INTO Pholus_Scan ("Info", "Time", "MAC", "IP_v4_or_v6", "Record_Type", "Value", "Extra") VALUES (?, ?, ?, ?, ?, ?, ?)""", params) 
+
+        else:
+            file_print('[', timeNow(), '] Scan: Pholus FAIL - check logs') 
 
 
 #-------------------------------------------------------------------------------
@@ -2276,6 +2343,10 @@ def append_line_to_file (pPath, pText):
 
 #-------------------------------------------------------------------------------
 def send_email (pText, pHTML):
+
+    # Print more info for debugging if PRINT_LOG enabled
+    print_log ('REPORT_TO: ' + hide_email(str(REPORT_TO)) + '  SMTP_USER: ' + hide_email(str(SMTP_USER))) 
+
     # Compose email
     msg = MIMEMultipart('alternative')
     msg['Subject'] = 'Pi.Alert Report'
@@ -2287,14 +2358,13 @@ def send_email (pText, pHTML):
     # Send mail
     smtp_connection = smtplib.SMTP (SMTP_SERVER, SMTP_PORT)
     smtp_connection.ehlo()
-#    smtp_connection.starttls()
+
-#    smtp_connection.ehlo()
-#    smtp_connection.login (SMTP_USER, SMTP_PASS)
     if not SafeParseGlobalBool("SMTP_SKIP_TLS"):
         smtp_connection.starttls()
         smtp_connection.ehlo()
     if not SafeParseGlobalBool("SMTP_SKIP_LOGIN"):
         smtp_connection.login (SMTP_USER, SMTP_PASS)
+
     smtp_connection.sendmail (REPORT_FROM, REPORT_TO, msg.as_string())
     smtp_connection.quit()
 
@@ -2440,6 +2510,8 @@ def publish_sensor(client, sensorConf):
 
     global mqtt_sensors         
 
+    file_print("        Estimated delay:", (len(mqtt_sensors) * int(MQTT_DELAY_SEC)))
+
     message = '{ \
                 "name":"'+ sensorConf.deviceName +' '+sensorConf.sensorName+'", \
                 "state_topic":"system-sensors/'+sensorConf.sensorType+'/'+sensorConf.deviceId+'/state", \
@@ -2458,11 +2530,10 @@ def publish_sensor(client, sensorConf):
 
     # add the sensor to the global list to keep track of succesfully added sensors
     if publish_mqtt(client, topic, message):
-                                # hack - delay adding to the queue in case the process is 
+                                     # hack - delay adding to the queue in case the process is 
         time.sleep(MQTT_DELAY_SEC)   # restarted and previous publish processes aborted 
-                                # (it takes ~2s to update a sensor config on the broker)
+                                     # (it takes ~2s to update a sensor config on the broker)
         mqtt_sensors.append(sensorConf)
-        # print(len(mqtt_sensors))
 
 #-------------------------------------------------------------------------------
 def mqtt_create_client():
@@ -2571,24 +2642,6 @@ def mqtt_start():
     # time.sleep(10)
 
 
-# #-------------------------------------------------------------------------------
-def start_mqtt_thread ():
-    # start a MQTT thread loop which will continuously report on devices to the broker
-        # daemon=True - makes sure the thread dies with the process if interrupted
-    global mqtt_thread_up
-
-    # flag to check if thread is running
-    mqtt_thread_up = True
-
-    file_print("    Starting MQTT sending")
-    x = threading.Thread(target=mqtt_start, args=(1,), daemon=True) 
-    # start_sending_mqtt(client)
-
-    file_print("    Threading: Starting MQTT thread")
-
-    x.start()
-
-
 #===============================================================================
 # DB
 #===============================================================================
@@ -2863,6 +2916,65 @@ def get_all_devices():
     closeDB()
     return row
 
+
+#-------------------------------------------------------------------------------
+def hide_email(email):
+    m = email.split('@')
+    return f'{m[0][0]}{"*"*(len(m[0])-2)}{m[0][-1] if len(m[0]) > 1 else ""}@{m[1]}'
+
+# Test
+print(hide_email('emailsecreto@gmail.com'))
+
+#-------------------------------------------------------------------------------
+def runSchedule():
+
+    global last_next_pholus_schedule
+    global last_pholus_scheduled_run
+    global last_next_pholus_schedule_used
+
+    result = False 
+
+    # datetime.now() - timedelta(days=1)
+    if last_pholus_scheduled_run == 0:
+        # last_pholus_scheduled_run =  datetime.datetime.fromtimestamp(pd.Timestamp(year = 2000, month = 1, day = 1, hour = 1, second = 1, tz = TIMEZONE))
+        last_pholus_scheduled_run =  (datetime.datetime.now(tz) - timedelta(days=365)).replace(microsecond=0)
+
+    nowTime = datetime.datetime.now(tz).replace(microsecond=0)
+
+
+    file_print("now                      : ", nowTime.isoformat(), "Type: ", type(nowTime))
+    file_print("last_pholus_scheduled_run: ", last_pholus_scheduled_run.isoformat(), "Type: ", type(last_pholus_scheduled_run))
+    file_print("last_next_pholus_schedule: ", last_next_pholus_schedule.isoformat(), "Type: ", type(last_next_pholus_schedule))
+
+
+    file_print("nowTime > last_next_pholus_schedule: ", nowTime > last_next_pholus_schedule)
+    file_print("last_pholus_scheduled_run < last_next_pholus_schedule: ", last_pholus_scheduled_run < last_next_pholus_schedule)
+
+
+    if nowTime > last_next_pholus_schedule and last_pholus_scheduled_run < last_next_pholus_schedule:
+        file_print("run: YES")
+        last_next_pholus_schedule_used = True
+        result = True
+    else:
+        file_print("run: NO")
+
+
+
+    
+
+    # file_print("last_next_pholus_schedule lastRunDateTime: ",   
+    
+    #  Debug
+
+    
+    if last_next_pholus_schedule_used:
+        last_next_pholus_schedule_used = False
+        last_next_pholus_schedule = schedule.next()    
+
+    file_print("runSchedule  n   : ", last_next_pholus_schedule.isoformat())    
+
+    return result
+
 #===============================================================================
 # BEGIN
 #===============================================================================

front/deviceDetails.php

@@ -119,13 +119,13 @@ if ($_SESSION["login"] != 1)
 
               <div class="btn-group pull-right">
                 <button type="button" class="btn btn-default"  style="padding: 10px; min-width: 30px;"
-                  id="btnPrevious" onclick="previousRecord()"> <i class="fa fa-chevron-left"></i> </button>
+                  id="btnPrevious" onclick="recordSwitch('prev')"> <i class="fa fa-chevron-left"></i> </button>
 
                 <div class="btn pa-btn-records"  style="padding: 10px; min-width: 30px; margin-left: 1px;"
                   id="txtRecord"     > 0 / 0 </div>
 
                 <button type="button" class="btn btn-default"  style="padding: 10px; min-width: 30px; margin-left: 1px;"
-                  id="btnNext"     onclick="nextRecord()"> <i class="fa fa-chevron-right"></i> </button>
+                  id="btnNext"     onclick="recordSwitch('next')"> <i class="fa fa-chevron-right"></i> </button>
               </div>
             </ul>
 
@@ -666,8 +666,8 @@ if ($ENABLED_DARKMODE === True) {
     return params.mac
   }  
 
-  mac                     = getMac()
+  mac                     = getMac()  // can also be rowID!! not only mac 
-  var devicesList         = [];
+  var devicesList         = [];   // this will contain a list the database row IDs of the devices ordered by the position displayed in the UI
   var pos                 = -1;
   var parPeriod           = 'Front_Details_Period';
   var parTab              = 'Front_Details_Tab';
@@ -1173,8 +1173,11 @@ function getDeviceData (readAllData=false) {
   // stop timer
   stopTimerRefreshData();
 
+  // console.log("getDeviceData mac: ", mac)
+
   // Check MAC
   if (mac == '') {
+    console.log("getDeviceData mac AA: ", mac)
     return;
   }
 
@@ -1326,9 +1329,9 @@ function getDeviceData (readAllData=false) {
       }
 
       // Check if device is part of the devicesList      
-      pos = devicesList.indexOf (deviceData['rowid']);
+      pos = devicesList.findIndex(item => item.rowid == deviceData['rowid']);      
       if (pos == -1) {
-        devicesList =[deviceData['rowid']];
+        devicesList.push({"rowid" : deviceData['rowid'], "mac" : deviceData['dev_MAC']});
         pos=0;
       }
     }
@@ -1363,45 +1366,44 @@ function getDeviceData (readAllData=false) {
 
 
 // -----------------------------------------------------------------------------
-function previousRecord () {
+// Left (prev) < > (next) Right toggles at the top right of device details to 
+// cycle between devices
+function recordSwitch(direction) {
+
+  var recordToSave = null;
+
+  // update the global position in the devices list variable 'pos'
+  if(direction == "next")
+  {
+    // Next Record
+    if (pos < (devicesList.length-1) ) {
+      pos++;
+    }
+  }else if (direction == "prev")
+  {
+    if (pos > 0) {
+      pos--;
+    }
+  }
+
+  // get new mac from the devicesList. Don't change to the commented out line below, the mac query string in the URL isn't updated yet!
+  // mac = params.mac;
+  mac = devicesList[pos].mac.toString();
+  
   // Save Changes
   if ( ! document.getElementById('btnSave').hasAttribute('disabled') ) {
-    setDeviceData (previousRecord);
+    setDeviceData (direction, recordSwitch);    
-    return;
   }
   
-  // Previous Record
+  getDeviceData (true); 
-  if (pos > 0) {
-    pos--;
-    mac = devicesList[pos].toString();
-    getDeviceData (true);
-  }
-}
 
-// -----------------------------------------------------------------------------
-function nextRecord () {
-  // Save Changes
-  if ( ! document.getElementById('btnSave').hasAttribute('disabled') ) {
-    setDeviceData (nextRecord);
-    return;
-  }
-
-  // get new mac
-  mac = params.mac;
   // reload current tab
   reloadTab()
 
-  // Next Record
-  if (pos < (devicesList.length-1) ) {
-    pos++;
-    mac = devicesList[pos].toString();
-    getDeviceData (true);
-  }
 }
 
-
 // -----------------------------------------------------------------------------
-function setDeviceData (refreshCallback='') {
+function setDeviceData (direction='', refreshCallback='') {
   // Check MAC
   if (mac == '') {
     return;
@@ -1441,7 +1443,7 @@ function setDeviceData (refreshCallback='') {
 
     // Callback fuction
     if (typeof refreshCallback == 'function') {
-      refreshCallback();
+      refreshCallback(direction);
     }
   });
 }
@@ -1621,8 +1623,8 @@ function initializeTabsNew () {
 
 function loadPholus()
 {
-    console.log(mac)
+    // console.log(mac)
-    console.log('php/server/devices.php?action=getPholus&mac='+ mac)
+    // console.log('php/server/devices.php?action=getPholus&mac='+ mac)
     $.get('php/server/devices.php?action=getPholus&mac='+ mac, function(data) {
       
       data = sanitize(data);      
@@ -1649,7 +1651,7 @@ function loadPholus()
       }
       else
       {
-        console.log("else")
+        // console.log("else")
         $("#tablePholusPlc").show();
         $(".deviceSpecific").remove();
       }        
@@ -1665,14 +1667,9 @@ window.onload = function async()
 
 function reloadTab()
 {
-  // Get the value of "some_key" in eg "https://example.com/?some_key=some_value"
-  mac = getMac(); // "some_value"
-
-  // console.log("aaAAAAAAAAAaa:"+my_mac)
   // load tab data only when needed (tab change)
   if(getCache("activeDevicesTab") == "tabPholus")
   {
-    console.log("herea")
     loadPholus();
   }
 }

front/devices.php

@@ -350,16 +350,35 @@ function initializeDatatable () {
     
   $('#tableDevices').on( 'order.dt', function () {
     setParameter (parTableOrder, JSON.stringify (table.order()) );
-    setCookie ('devicesList',JSON.stringify (table.column(12, { 'search': 'applied' }).data().toArray()) );
+    setCookie ('devicesList', getDevicesFromTable(table) );
   } );
 
   $('#tableDevices').on( 'search.dt', function () {
-    setCookie ('devicesList', JSON.stringify (table.column(12, { 'search': 'applied' }).data().toArray()) );
+    setCookie ('devicesList', getDevicesFromTable(table) ); 
   } );
 
 };
 
 
+// -----------------------------------------------------------------------------
+// Gets a JSON list of rowID and mac from the displayed table in the UI
+function getDevicesFromTable(table)
+{
+  rowIDs = table.column(12, { 'search': 'applied' }).data().toArray()  // rowID is in hidden column 12
+  rowMACs = table.column(10, { 'search': 'applied' }).data().toArray() // MAC is in hidden column 10
+
+  result = []
+
+  rowIDs.map(function(rowID, index){
+    result.push({"rowid": rowID, "mac":rowMACs[index]})
+  })
+  
+  // console.log(rowIDs)
+  // console.log(result)  
+
+  return JSON.stringify (result)
+}
+
 // -----------------------------------------------------------------------------
 function getDevicesTotals () {
   // stop timer

front/php/server/devices.php

@@ -910,11 +910,17 @@ function getPholus() {
   // SQL
   $mac = $_REQUEST['mac']; 
 
+  if ($mac == "Internet") // Not performing data lookup for router (improvement idea for later maybe)
+  {
+    echo "false";
+    return;
+  }
+
   if (false === filter_var($mac , FILTER_VALIDATE_MAC)) {
     throw new Exception('Invalid mac address');
   }
   else{
-    $sql = 'SELECT * from Pholus_Scan where MAC ="'.$mac.'"';
+    $sql = 'SELECT * from Pholus_Scan where MAC ="'.$mac.'" and Record_Type not in ("Question")';
 
     // array 
     $tableData = array();

front/php/server/util.php

@@ -273,7 +273,7 @@ function saveSettings()
 
   displayMessage("<br/>Settings saved to the <code>".$config_file."</code> file.  
     <br/><br/>Backup of the previous ".$config_file." created here: <br/><br/><code>".$new_name."</code><br/><br/>
-    <b>Note:</b> Wait <b>20s</b> for the changes to reflect in the UI.", 
+    <b>Note:</b> Wait <b>5s</b> for the changes to reflect in the UI.", 
     FALSE, TRUE, TRUE, TRUE);    
 
 }

front/php/templates/language/en_us.php

@@ -430,11 +430,11 @@ $lang['en_us'] = array(
 'SCAN_SUBNETS_name' => 'Subnets to scan',
 'SCAN_SUBNETS_description' => '
 
-The scan time itself depends on the number of IP addresses to check. 
+The arp-scan time itself depends on the number of IP addresses to check. 
 The number of IPs to check depends on the <a target="_blank" href="https://www.calculator.net/ip-subnet-calculator.html">network mask</a> you set here. 
 For example, a <code>/24</code> mask results in 256 IPs to check, where as a <code>/16</code> 
 mask checks around 65,536. Every IP takes a couple seconds. This means that with an incorrect configuration 
-the scan will take hours to complete instead of seconds.
+the arp-scan will take hours to complete instead of seconds.
 <ol>
 <li>Specify the network mask. For example, the filter <code>192.168.1.0/24</code> covers IP ranges 192.168.1.0 to 192.168.1.255.</li>
 <li>Run <code>iwconfig</code> in your container to find your interface name(s) (e.g.: <code>eth0</code>, <code>eth1</code>)</li>
@@ -451,7 +451,7 @@ the scan will take hours to complete instead of seconds.
 'INCLUDED_SECTIONS_name' => 'Notify on',
 'INCLUDED_SECTIONS_description' => 'Specifies which events trigger notifications. Remove the event type(s) you don\'t want to get notified on. This setting overrides device-specific settings in the UI. (CTRL + Click to select / deselect).',
 'SCAN_CYCLE_MINUTES_name' => 'Scan cycle delay',
-'SCAN_CYCLE_MINUTES_description' => 'The delay between scans. If using arp-scan, the scan time itself depends on the number of IP addresses to check. This is influenced by the network mask set in the <code>SCAN_SUBNETS</code> setting at the top. Every IP takes a couple seconds to scan.',
+'SCAN_CYCLE_MINUTES_description' => 'The delay between scans. If using arp-scan, the scan time itself depends on the number of IP addresses to check. This is influenced by the network mask set in the <a href="#SCAN_SUBNETS"><code>SCAN_SUBNETS</code> setting</a> at the top. Every IP takes a couple seconds to scan.',
 'DAYS_TO_KEEP_EVENTS_name' => 'Delete events older than',
 'DAYS_TO_KEEP_EVENTS_description' => 'This is a maintenance setting. This specifies the number of days worth of event entries that will be kept. All older events will be deleted periodically.',
 'REPORT_DASHBOARD_URL_name' => 'Pi.Alert URL',
@@ -550,9 +550,19 @@ the scan will take hours to complete instead of seconds.
 
 // Pholus
 'PHOLUS_ACTIVE_name' => 'Enable Pholus scan',
-'PHOLUS_ACTIVE_description' => '<a href="https://github.com/jokob-sk/Pi.Alert/tree/main/pholus" target="_blank" >Pholus</a> is a sniffing tool to discover additional information about the devices on the network, including the device name. Please be aware it can spam the network with unnecessary traffic.',
+'PHOLUS_ACTIVE_description' => '<a href="https://github.com/jokob-sk/Pi.Alert/tree/main/pholus" target="_blank" >Pholus</a> is a sniffing tool to discover additional information about the devices on the network, including the device name. Please be aware it can spam the network with unnecessary traffic. Depends on the <a href="#SCAN_SUBNETS"><code>SCAN_SUBNETS</code> setting</a>.',
 'PHOLUS_TIMEOUT_name' => 'Pholus timeout',
-'PHOLUS_TIMEOUT_description' => 'How long (s) should Pholus be sniffing the network. Only used if an <code>(unknown)</code> device is found. The longer you leave it on, the more likely devices would broadcast more info.',
+'PHOLUS_TIMEOUT_description' => 'How long (s) should Pholus be sniffing the network. Only used if an <code>(unknown)</code> device is found. The longer you leave it on, the more likely devices would broadcast more info. This timeout adds to the time it takes to perform an arp-scan on your network',
+'PHOLUS_FORCE_name' => 'Force scan',
+'PHOLUS_FORCE_description' => 'Force scan every network scan, even if there are no <code>(unknown)</code> devices. Be careful enabling this as the sniffing can easily flood your network.',
+'PHOLUS_DAYS_DATA_name' => 'Data retention',
+'PHOLUS_DAYS_DATA_description' => 'How many days of Pholus scan entries should be kept (gloablly, not device specific!). The <a href="/maintenance.php#tab_Logging">pialert_pholus.log</a> file is not touched.',
+'PHOLUS_RUN_name' => 'Run on schedule',
+'PHOLUS_RUN_description' => 'Enable a regular Pholus scan / sniff on your network.',
+'PHOLUS_RUN_TIMEOUT_name' => 'Scheduled run timeout',
+'PHOLUS_RUN_TIMEOUT_description' => 'The timeout (s) for the scheduled Pholus scan.',
+'PHOLUS_RUN_SCHD_name' => 'Schedule',
+'PHOLUS_RUN_SCHD_description' => 'Schedule in cron format. Make sure you enter the schedule in the correct format (e.g. validate your format on <a href="#" onlick="window.open("https://crontab.guru/#" + $(\'#PHOLUS_RUN_SCHD\').val(),replace(\' \', \'_\') , "_blank")" target="_blank">crontab.guru</a>). Will be run NEXT time the time passes.',
 
 );
 

front/settings.php

@@ -142,9 +142,18 @@ $db->close();
               elseif ($set['Type'] == 'selecttext')
               {
                 $input = '<select class="form-control" name="'.$set['Code_Name'].'" id="'.$set['Code_Name'].'">';                 
+                
+                $values = createArray($set['Value']);
                 $options = createArray($set['Options']);
+
                 foreach ($options as $option) {              
-                  $input = $input.'<option value="'.$option.'">'.$option.'</option>';
+                  $selected = "";
+
+                  if( in_array( $option , $values) == true) {
+                    $selected = "selected";
+                  }
+
+                  $input = $input.'<option value="'.$option.'" '.$selected.'>'.$option.'</option>';
                 }                
                 $input = $input.'</select>';
               }
@@ -152,9 +161,19 @@ $db->close();
               elseif ($set['Type'] == 'selectinteger')
               {
                 $input = '<select class="form-control" name="'.$set['Code_Name'].'" id="'.$set['Code_Name'].'">';                 
+
+                $values = createArray($set['Value']);
                 $options = createArray($set['Options']);
+
                 foreach ($options as $option) {   
-                  $input = $input.'<option value="'.$option.'">'.$option.'</option>';
+                  
+                  $selected = "";
+
+                  if( in_array( $option , $values) == true) {
+                    $selected = "selected";
+                  }
+
+                  $input = $input.'<option value="'.$option.'" '.$selected.'>'.$option.'</option>';
                 }                
                 $input = $input.'</select>';
               }
@@ -162,6 +181,7 @@ $db->close();
               elseif ($set['Type'] == 'multiselect')
               {
                 $input = '<select class="form-control" name="'.$set['Code_Name'].'" id="'.$set['Code_Name'].'" multiple>';  
+
                 $values = createArray($set['Value']);
                 $options = createArray($set['Options']);
 
@@ -242,7 +262,7 @@ $db->close();
 <script>
 
   // number of settings has to be equal to
-  var settingsNumber = 48;
+  var settingsNumber = 53;
   
   if(<?php echo count($settings)?> != settingsNumber) 
   {